feat: enable index resumption and enforce directory creation
The command now supports reopening existing indexes instead of failing when the output file exists. Control flow branches between opening an existing index and constructing a new one, moving configuration setup exclusively to the creation path. Directory existence is enforced upfront with proper I/O error propagation. The --force flag retains its original semantics by removing the target directory before proceeding with a fresh build.
This commit is contained in:
Eric Coissac
+83
-47
@@ -1,39 +1,6 @@
|
||||
# the name by which the project can be referenced within Serena
|
||||
# the name by which the project can be referenced within Serena/when chatting with the LLM.
|
||||
project_name: "obikmer"
|
||||
|
||||
|
||||
# list of languages for which language servers are started; choose from:
|
||||
# al angular ansible bash clojure
|
||||
# cpp cpp_ccls crystal csharp csharp_omnisharp
|
||||
# dart elixir elm erlang fortran
|
||||
# fsharp go groovy haskell haxe
|
||||
# hlsl html java json julia
|
||||
# kotlin lean4 lua luau markdown
|
||||
# matlab msl nix ocaml pascal
|
||||
# perl php php_phpactor powershell python
|
||||
# python_jedi python_ty r rego ruby
|
||||
# ruby_solargraph rust scala scss solidity
|
||||
# svelte swift systemverilog terraform toml
|
||||
# typescript typescript_vts vue yaml zig
|
||||
# (This list may be outdated. For the current list, see values of Language enum here:
|
||||
# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py
|
||||
# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
|
||||
# Note:
|
||||
# - For C, use cpp
|
||||
# - For JavaScript, use typescript
|
||||
# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
|
||||
# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
|
||||
# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
|
||||
# - For Free Pascal/Lazarus, use pascal
|
||||
# Special requirements:
|
||||
# Some languages require additional setup/installations.
|
||||
# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
|
||||
# When using multiple languages, the first language server that supports a given file will be used for that file.
|
||||
# The first language is the default language and the respective language server will be used as a fallback.
|
||||
# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
|
||||
languages:
|
||||
- rust
|
||||
|
||||
# the encoding used by text files in the project
|
||||
# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
|
||||
encoding: "utf-8"
|
||||
@@ -55,23 +22,19 @@ ignore_all_files_in_gitignore: true
|
||||
|
||||
# advanced configuration option allowing to configure language server-specific options.
|
||||
# Maps the language key to the options.
|
||||
# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available.
|
||||
# No documentation on options means no options are available.
|
||||
# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
|
||||
# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
|
||||
ls_specific_settings: {}
|
||||
|
||||
# list of additional workspace folder paths for cross-package reference support (e.g. in monorepos).
|
||||
# Paths can be absolute or relative to the project root.
|
||||
# Each folder is registered as an LSP workspace folder, enabling language servers to discover
|
||||
# symbols and references across package boundaries.
|
||||
# Currently supported for: TypeScript.
|
||||
# Example:
|
||||
# additional_workspace_folders:
|
||||
# - ../sibling-package
|
||||
# - ../shared-lib
|
||||
additional_workspace_folders: []
|
||||
|
||||
# list of additional paths to ignore in this project.
|
||||
# Same syntax as gitignore, so you can use * and **.
|
||||
# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
|
||||
# Example:
|
||||
# ignored_paths:
|
||||
# - "examples/**"
|
||||
# - ".worktrees/**"
|
||||
# - "**/bin/**"
|
||||
# - "**/obj/**"
|
||||
# Note: global ignored_paths from serena_config.yml are also applied additively.
|
||||
ignored_paths: []
|
||||
|
||||
@@ -131,3 +94,76 @@ read_only_memory_patterns: []
|
||||
# Extends the list from the global configuration, merging the two lists.
|
||||
# Example: ["_archive/.*", "_episodes/.*"]
|
||||
ignored_memory_patterns: []
|
||||
|
||||
# list of additional workspace folder paths for cross-package reference support.
|
||||
# Paths can be absolute or relative to the project root.
|
||||
# Each folder is registered as an LSP workspace folder, enabling language servers to discover
|
||||
# symbols and references across package boundaries, but these folders are not indexed by Serena,
|
||||
# i.e. the respective symbols will not be found using Serena's symbol search tools.
|
||||
# Example:
|
||||
# additional_workspace_folders:
|
||||
# - ../sibling-package
|
||||
# - ../shared-lib
|
||||
ls_additional_workspace_folders: []
|
||||
|
||||
# list of language servers to start when using the LSP backend; choose from:
|
||||
# ada al angular ansible bash
|
||||
# bsl clojure cpp cpp_ccls crystal
|
||||
# csharp csharp_omnisharp cue dart deno
|
||||
# elixir elm erlang fortran fsharp
|
||||
# gdscript gleam go groovy haskell
|
||||
# haxe hlsl html java json
|
||||
# julia kotlin latex lean4 lua
|
||||
# luau markdown matlab msl nextflow
|
||||
# nix ocaml pascal perl php
|
||||
# php_phpactor php_phpantom powershell python python_basedpyright
|
||||
# python_jedi python_pyrefly python_ty qml r
|
||||
# rego ruby ruby_solargraph rust scala
|
||||
# scss solidity svelte swift systemverilog
|
||||
# terraform toml typescript typescript_vts vue
|
||||
# wolfram yaml zig
|
||||
# (This list may be outdated; generated with scripts/print_language_list.py;
|
||||
# For the current list, see values of the LanguageServerId enum here:
|
||||
# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
|
||||
# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
|
||||
# Note:
|
||||
# - For C, use cpp
|
||||
# - For JavaScript, use typescript
|
||||
# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
|
||||
# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
|
||||
# - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH)
|
||||
# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
|
||||
# - For Free Pascal/Lazarus, use pascal
|
||||
# Special requirements:
|
||||
# Some language servers require additional setup/installations.
|
||||
# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
|
||||
# When using multiple language servers, the first language server that supports a given file will be used for that file.
|
||||
# The first language server is the default language and the respective language server will be used as a fallback.
|
||||
# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
|
||||
language_servers:
|
||||
- rust
|
||||
|
||||
# list of workspace folder paths (LSP backend only).
|
||||
# These folders will be used to build up Serena's symbol index.
|
||||
# Paths must be within the project root and should thus be relative to the project root.
|
||||
# Furthermore, the paths should not be filtered by ignore settings.
|
||||
# Default setting: The entire project root folder (".") is considered.
|
||||
# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
|
||||
# ls_workspace_folders:
|
||||
# - "./subproject1"
|
||||
# - "./subproject2"
|
||||
ls_workspace_folders:
|
||||
- .
|
||||
|
||||
# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
|
||||
# the command runs in the project root directory and is only executed if the project is trusted
|
||||
# (see trusted_project_path_patterns in the global configuration).
|
||||
# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
|
||||
# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
|
||||
# backstop for non-terminating commands; on expiry the process is killed and activation continues.
|
||||
# example: activation_command: "npx nx run-many -t build"
|
||||
activation_command:
|
||||
|
||||
# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
|
||||
# must be a positive number.
|
||||
activation_command_timeout: 180.0
|
||||
|
||||
@@ -90,7 +90,7 @@ Je continue à poser mes questions et à guider la discussion.
|
||||
- Diagnostics LSP (erreurs de compilation) : `mcp__serena__get_diagnostics_for_file`
|
||||
- Vue d'ensemble d'un fichier : `mcp__serena__get_symbols_overview`
|
||||
- Modifier le corps d'une fonction/impl : `mcp__serena__replace_symbol_body`
|
||||
- Ne pas utiliser `cclsp` quand serena couvre le besoin
|
||||
- Ne pas utiliser `cclsp` ni `rust-analyzer-mcp` quand serena couvre le besoin — `rust-analyzer-mcp` en repli ponctuel (hover, diagnostics workspace) si serena ne couvre pas le cas
|
||||
|
||||
**Analyse architecturale → jcodemunch**
|
||||
- Hotspots, couplage, dead code, dépendances entre modules
|
||||
@@ -101,3 +101,8 @@ Je continue à poser mes questions et à guider la discussion.
|
||||
|
||||
**Documentation de crates → context7**
|
||||
- Toujours consulter avant d'utiliser une API de bibliothèque externe
|
||||
|
||||
**Recherche sémantique dans la doc → qmd**
|
||||
- Les deux arbres de doc sont indexés comme collections `qmd` : `obikmer-devdoc` (`DevDocMD/`, 57 fichiers) et `obikmer-userdoc` (`UserDocMD/`, 25 fichiers)
|
||||
- Interroger via les outils MCP `query`/`get`/`multi_get`/`status`, ou en CLI (`qmd search`/`qmd query -c obikmer-devdoc "..."`)
|
||||
- Après ajout/modification de fichiers Markdown dans `DevDocMD/` ou `UserDocMD/`, relancer `qmd embed` pour que l'index reste à jour (les collections ne se rafraîchissent pas automatiquement)
|
||||
|
||||
+30
-2
@@ -722,14 +722,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="/implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -967,6 +967,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1242,10 +1270,10 @@
|
||||
<h1 id="coverage-architectureindex_architecturemd">Coverage: architecture/index_architecture.md</h1>
|
||||
<h2 id="code-couvert">Code couvert</h2>
|
||||
<ul>
|
||||
<li><code>obilayeredmap/src/layer.rs</code> — Layer<D>, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix</li>
|
||||
<li><code>obilayeredmap/src/mphf_layer.rs</code> — MphfLayer, EvidenceKind (Exact / Approx), LayerEvidence enum</li>
|
||||
<li><code>obilayeredmap/src/map.rs</code> — LayeredMap<D></li>
|
||||
<li><code>obilayeredmap/src/meta.rs</code> — LayerMeta, PartitionMeta</li>
|
||||
<li><code>obikindex/src/layer/typed_layer.rs</code> — Layer<D>, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix</li>
|
||||
<li><code>obikindex/src/layer/mphf_layer.rs</code> — MphfLayer, EvidenceKind (Exact / Approx), LayerEvidence enum</li>
|
||||
<li><code>obikindex/src/layer/map.rs</code> — LayeredMap<D></li>
|
||||
<li><code>obikindex/src/layer/meta.rs</code> — LayerMeta, PartitionMeta</li>
|
||||
<li><code>obikindex/src/meta.rs</code> — IndexConfig (kmer_size, n_bits, with_counts, evidence, block_bits), IndexMeta</li>
|
||||
<li><code>obikindex/src/index.rs</code> — KmerIndex, build_layers</li>
|
||||
<li><code>obicompactvec/src/</code> — PersistentCompactIntMatrix, PersistentBitMatrix (DataStore implementations)</li>
|
||||
|
||||
@@ -731,14 +731,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -976,6 +976,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -729,14 +729,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -974,6 +974,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -731,14 +731,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -976,6 +976,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1532,7 +1560,7 @@
|
||||
<td>Replace <code>available_parallelism()</code> with per-NUMA core count for spawn criterion</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>obikpartitionner/src/merge_layer.rs</code></td>
|
||||
<td><code>obikindex/src/partition/merge_layer.rs</code></td>
|
||||
<td>No change — <code>merge_partition</code> already works inside any Rayon context</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -1540,7 +1568,7 @@
|
||||
<td>No change — <code>par_iter</code> and <code>current_num_threads</code> are pool-context-aware</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>obikpartitionner/src/partition.rs</code></td>
|
||||
<td><code>obikindex/src/partition/partition.rs</code></td>
|
||||
<td>No change — same reason</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1243,7 +1271,7 @@
|
||||
<h2 id="code-couvert">Code couvert</h2>
|
||||
<ul>
|
||||
<li><code>obikmer/src/cmd/query.rs</code> — commande query, format de sortie</li>
|
||||
<li><code>obikpartitionner/src/query_layer.rs</code> — routage de la requête à travers les partitions</li>
|
||||
<li><code>obikindex/src/partition/query_layer.rs</code> — routage de la requête à travers les partitions</li>
|
||||
<li><code>obiread/src/lib.rs</code> — lecture des séquences d'entrée pour la requête</li>
|
||||
</ul>
|
||||
<h2 id="notes">Notes</h2>
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1527,7 +1555,7 @@
|
||||
</ul>
|
||||
<hr />
|
||||
<h2 id="algorithm">Algorithm</h2>
|
||||
<p>The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside <code>process_chunk</code> (<code>query.rs</code>); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (<code>obikpartitionner::query_partition_with</code>), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.</p>
|
||||
<p>The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside <code>process_chunk</code> (<code>query.rs</code>); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (<code>obikindex::partition::query_partition_with</code>), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.</p>
|
||||
<div class="highlight"><pre><span></span><code>for each chunk of sequences (parallel workers via obipipeline, one call to process_chunk):
|
||||
build QueryBatch (QueryBatch::from_records):
|
||||
decompose all sequences into superkmers (SuperKmerIter) — construction only,
|
||||
@@ -1602,9 +1630,9 @@
|
||||
<p>The <code>-z</code> CLI option overrides the index metadata value. A higher z increases stringency (lower FP, some true positives may be discarded at sequence ends); a lower z increases sensitivity.</p>
|
||||
<hr />
|
||||
<h2 id="layer-lookup-mphflayerfind">Layer lookup: <code>MphfLayer::find</code></h2>
|
||||
<p><code>MphfLayer::open(dir, mode: &IndexMode)</code> receives the mode from <code>PartitionMeta</code> — no per-layer file is read. The caller (<code>QueryLayer</code>) never chooses the dispatch path: it is fixed at open time by <code>LayerEvidence</code>. See <a href="../../implementation/obilayeredmap/">obilayeredmap</a> for the full <code>find</code> / <code>find_strict</code> API.</p>
|
||||
<p><code>MphfLayer::open(dir, mode: &IndexMode)</code> receives the mode from <code>PartitionMeta</code> — no per-layer file is read. The caller (<code>QueryLayer</code>) never chooses the dispatch path: it is fixed at open time by <code>LayerEvidence</code>. See <a href="../../implementation/layer_tier/">obikindex::layer</a> for the full <code>find</code> / <code>find_strict</code> API.</p>
|
||||
<h3 id="querylayer-variant-selection"><code>QueryLayer</code> variant selection</h3>
|
||||
<p><code>QueryLayer::open</code> (<code>obikpartitionner/src/query_layer.rs:28-45</code>) only ever returns two variants — <code>Presence</code> or <code>Count</code>, checked in this order:</p>
|
||||
<p><code>QueryLayer::open</code> (<code>obikindex/src/partition/query_layer.rs:28-45</code>) only ever returns two variants — <code>Presence</code> or <code>Count</code>, checked in this order:</p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
@@ -1774,7 +1802,7 @@ Fix direction: restructure <code>query</code>'s pipe with an initial <code>Flat<
|
||||
<p><strong>3. Chunk-size memory formula ignores <code>n_genomes</code>.</strong>
|
||||
<code>chunk_bytes = available_memory_bytes() / (n_workers * 16)</code> (<code>query.rs:407-414</code>) assumes a fixed ~8–16× overhead per raw input byte. But <code>KmerResults::new</code> (<code>query.rs:165-179</code>) allocates <code>data: Vec<u32></code> sized <code>total_kmers_in_chunk × n_genomes</code> — dense, <strong>for every k-mer position in the chunk, hit or not</strong> — plus <code>win_min</code> and (with <code>--detail</code>) <code>cov</code>, same scaling. Real per-chunk memory is <code>O(n_genomes)</code>, not constant; the formula doesn't know <code>n_genomes</code> at all. This is the direct cause of the OOM kill on indexes with many reference genomes.</p>
|
||||
<p><strong>4. MPHF lookup and matrix-row fetch are fused, not staged.</strong>
|
||||
<code>QueryLayer::find_into</code> (<code>obikpartitionner/src/query_layer.rs:48-67</code>) does the MPHF <code>find</code> <em>and</em> the <code>fill_row</code> matrix read in one call per k-mer, inside a single-threaded loop (<code>query_partition_with</code>). There is no separation between "is this k-mer indexed" (cheap, <code>O(1)</code>, independent of <code>n_genomes</code>) and "what are its per-genome values" (the expensive, <code>n_genomes</code>-scaling part).</p>
|
||||
<code>QueryLayer::find_into</code> (<code>obikindex/src/partition/query_layer.rs:48-67</code>) does the MPHF <code>find</code> <em>and</em> the <code>fill_row</code> matrix read in one call per k-mer, inside a single-threaded loop (<code>query_partition_with</code>). There is no separation between "is this k-mer indexed" (cheap, <code>O(1)</code>, independent of <code>n_genomes</code>) and "what are its per-genome values" (the expensive, <code>n_genomes</code>-scaling part).</p>
|
||||
<p><strong>5. Dereplication should happen at k-mer granularity, directly — not via an intermediate superkmer-level dedup.</strong>
|
||||
<code>QueryBatch::from_records</code> currently dereplicates at the <em>superkmer</em> level (<code>HashMap<RoutableSuperKmer, Vec<SKDesc>></code>, <code>query.rs:112</code>). This misses redundancy between k-mers shared by <em>different</em> superkmers (read overlaps, repeats, a SNP splitting an otherwise-identical run). Superkmer <em>construction</em> (<code>SuperKmerIter</code>) stays mandatory — it is the mechanism that computes minimizers/partition routing, not an optional dedup layer — but the dedup structure built on top of it should key directly on <code>CanonicalKmer</code>, in the same pass: <code>HashMap<CanonicalKmer, Vec<(seq_idx, pos)>></code>. This also means the MPHF <code>find</code> itself runs once per <strong>distinct</strong> k-mer instead of once per occurrence — a win independent of the matrix-fetch cost below.</p>
|
||||
<p><strong>6. Stage 1 output: bucket confirmed hits by layer, keyed by MPHF slot.</strong>
|
||||
@@ -1806,12 +1834,12 @@ Stage 2's output should be <code>HashMap<genome_idx, Vec<(seq_idx, positio
|
||||
<p>None of this is implemented yet — parked here as a coherent roadmap while the design is discussed further. Suggested dependency order: (1) I/O parallelism → (3) genome-aware chunk sizing → (4)–(9) staged/k-mer-deduped/NUMA-aware-partition-and-column-major/sparse query engine (larger refactor, biggest structural payoff — reuses <code>PartitionRunner</code> rather than inventing a new parallelism mechanism) → (2) parallel gzip (separate, orthogonal, tracked in chunkreader.md) → secondary diagnostics patterns.</p>
|
||||
<hr />
|
||||
<h2 id="implementation-plan">Implementation plan</h2>
|
||||
<p>Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current <code>query</code> output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning <code>obikmer</code>, <code>obikpartitionner</code>, and <code>obicompactvec</code> — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense <code>KmerResults</code>) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if <code>rapidgzip-rs</code> is validated (see <a href="../../implementation/chunkreader/#future-work--parallel-gzip-decompression-in-xopen">chunkreader.md</a>).</p>
|
||||
<p>Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current <code>query</code> output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning <code>obikmer</code>, <code>obikindex::partition</code>, and <code>obicompactvec</code> — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense <code>KmerResults</code>) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if <code>rapidgzip-rs</code> is validated (see <a href="../../implementation/chunkreader/#future-work--parallel-gzip-decompression-in-xopen">chunkreader.md</a>).</p>
|
||||
<p>Instrumentation is deliberately sequenced <em>before</em> the I/O fix (reordering the roadmap's own listed order), because every later phase's justification rests on a measurement ("to be measured, not assumed" appears throughout the roadmap above) — without it, phases 3–5 would be undertaken on faith.</p>
|
||||
<p>Performance measurement on the reference 192-core/8-NUMA machine is done by the project owner, not from this development environment (macOS, 16 cores — <code>PartitionRunner</code>'s NUMA pinning is Linux-only, so even phase 4's mechanism can't be functionally exercised for its actual purpose here). Each phase below is therefore written to be <em>self-measuring</em>: the debug-level logging it adds must be enough, on its own, to judge whether that phase's algorithmic choice paid off from a cluster run's logs, without needing to attach a profiler.</p>
|
||||
<h3 id="conventions-applied-to-every-phase-below">Conventions applied to every phase below</h3>
|
||||
<p><strong>Debug logging.</strong> Every phase that changes an algorithmic choice (not phase 0, which <em>is</em> the logging) adds <code>tracing::debug!</code>/<code>trace!</code> at points that let a cluster run's logs answer "did this help": counts, ratios, and timings that quantify the specific claim that phase makes — e.g. phase 3 must log how many MPHF <code>find</code> calls were saved by k-mer-level dedup (the whole justification for that phase), phase 4 must log per-column scan timings, phase 5 must log actual retained-memory / sparsity ratios achieved. Prefer one structured <code>debug!</code> per chunk (fields, not prose) over free-text — the cluster logs will be the only evidence available for judging these choices, so they need to be grep/awk-able, not just readable.</p>
|
||||
<p><strong>Unit tests.</strong> This project's convention (<code>obiread</code>, <code>obikseq</code>, <code>obidebruinj</code>, <code>obicompactvec</code>, <code>obilayeredmap</code>, <code>obiskio</code>, <code>obifastwrite</code>) is <code>#[cfg(test)] #[path = "tests/<name>.rs"] mod tests;</code> at the bottom of the source file, with the actual test code in a sibling <code>src/tests/<name>.rs</code>. Neither <code>obikmer</code> nor <code>obikpartitionner</code> (the two crates phases 3 and 5 touch most) currently have a <code>src/tests/</code> directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.</p>
|
||||
<p><strong>Unit tests.</strong> This project's convention (<code>obiread</code>, <code>obikseq</code>, <code>obidebruinj</code>, <code>obicompactvec</code>, <code>obikindex::layer</code>, <code>obiskio</code>, <code>obifastwrite</code>) is <code>#[cfg(test)] #[path = "tests/<name>.rs"] mod tests;</code> at the bottom of the source file, with the actual test code in a sibling <code>src/tests/<name>.rs</code>. Neither <code>obikmer</code> nor <code>obikindex::partition</code> (the two crates phases 3 and 5 touch most) currently have a <code>src/tests/</code> directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.</p>
|
||||
<p><strong>Workflow (<code>jj</code>).</strong> Work happens in a fresh <code>jj</code> commit, easy to abandon. <code>jj new</code> between phases is reasonable where it helps isolate a phase for review, but only when the working copy compiles at that point (project convention) — phase 3's internal sub-steps (batch dedup change, then <code>query_layer.rs</code> split, then the new return shape) will likely not each compile independently since they're one coupled change, so treat "commit boundary" and "plan phase boundary" as related but not forced to match 1:1; use judgement per phase rather than mechanically splitting on every bullet.</p>
|
||||
<h3 id="phase-0-instrumentation-prerequisite-for-measuring-every-later-phase">Phase 0 — Instrumentation (prerequisite for measuring every later phase)</h3>
|
||||
<p><strong>Goal</strong>: make core utilization, throughput, and per-stage timing visible on a real run, so phases 1–5 can be justified with numbers instead of assumption.</p>
|
||||
@@ -1854,26 +1882,26 @@ Stage 2's output should be <code>HashMap<genome_idx, Vec<(seq_idx, positio
|
||||
<li><code>obikmer/src/cmd/query.rs</code>:</li>
|
||||
<li>Replace <code>QueryBatch::from_records</code>'s dedup map (<code>HashMap<RoutableSuperKmer, Vec<SKDesc>></code>, current <code>query.rs:112</code>) with a per-partition <code>HashMap<CanonicalKmer, Vec<(seq_idx: u32, pos: u32)>></code>, built in the same <code>SuperKmerIter</code> pass: superkmer construction and partition routing (<code>part_idx</code> from the superkmer's minimizer hash) are unchanged, only the granularity of what gets deduplicated changes — each <code>CanonicalKmer</code> within a superkmer is inserted individually instead of the whole superkmer being the dedup key.</li>
|
||||
<li><strong>Verified</strong>: <code>CanonicalKmer</code> (<code>obikseq/src/kmer.rs:390</code>, <code>pub type CanonicalKmer = CanonicalKmerOf<KLen></code>) — the underlying <code>CanonicalKmerOf<L></code> derives <code>Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash</code> (<code>kmer.rs:269</code>). Usable as a <code>HashMap</code>/<code>HashSet</code> key as-is, no change needed.</li>
|
||||
<li><code>obikpartitionner/src/query_layer.rs</code>:</li>
|
||||
<li><code>obikindex/src/partition/query_layer.rs</code>:</li>
|
||||
<li>Split <code>QueryLayer::find_into</code> (<code>query_layer.rs:48-67</code>) into two methods: <code>find_slot(&self, kmer: CanonicalKmer) -> Option<usize></code> (MPHF only, no matrix touch) and keep <code>fill_row</code> as-is for phase 4 to call later.</li>
|
||||
<li>Replace <code>query_partition_with</code>'s inner loop (<code>query_layer.rs:103-113</code>) with a version that, for each unique <code>CanonicalKmer</code>, calls <code>find_slot</code> across the partition's layers (stopping at first hit, same as today), and instead of immediately filling a row, records <code>(layer_idx, slot)</code>.</li>
|
||||
<li>New return shape for the partition-level query, replacing today's <code>on_hit(sk_idx, kmer_idx, row)</code> callback: <code>HashMap<layer_idx, HashMap<slot, Vec<(seq_idx, pos)>>></code> (roadmap point 6) — built directly from the k-mer dedup map's <code>Vec<(seq_idx,pos)></code> values, keyed by the resolved slot instead of the k-mer.</li>
|
||||
<li><strong>This phase alone has no throughput benefit yet</strong> (matrix fetch still happens, just deferred) beyond the k-mer-level dedup itself (fewer MPHF calls when queries have overlapping/repeated k-mers) — its purpose is to produce the input phase 4 needs. Land phase 3+4 together, not phase 3 alone, per the "don't split 3–5 across releases" note above.</li>
|
||||
<li>Log, per chunk: total k-mer occurrences vs. unique <code>CanonicalKmer</code> count (the dedup ratio — the entire justification for this phase) and the resulting MPHF <code>find</code> call count. If the dedup ratio is close to <code>1.0</code> on real query data (little redundancy), that's the cluster run telling us this phase wasn't worth it — the logging needs to be able to say that, not just confirm the happy path.</li>
|
||||
<li><strong>Unit tests</strong>: create <code>obikmer/src/cmd/tests/query.rs</code> (new <code>src/tests/</code> dir for this crate, following the project's <code>#[cfg(test)] #[path = "tests/query.rs"] mod tests;</code> convention) and <code>obikpartitionner/src/tests/query_layer.rs</code> (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the <code>find_slot</code>/bucket-by-layer-and-slot construction against a small hand-built <code>QueryLayer</code> fixture, asserting the <code>(layer_idx, slot, seq_idx, pos)</code> tuples match what the old per-occurrence loop would have produced.</li>
|
||||
<li><strong>Unit tests</strong>: create <code>obikmer/src/cmd/tests/query.rs</code> (new <code>src/tests/</code> dir for this crate, following the project's <code>#[cfg(test)] #[path = "tests/query.rs"] mod tests;</code> convention) and <code>obikindex/src/partition/tests/query_layer.rs</code> (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the <code>find_slot</code>/bucket-by-layer-and-slot construction against a small hand-built <code>QueryLayer</code> fixture, asserting the <code>(layer_idx, slot, seq_idx, pos)</code> tuples match what the old per-occurrence loop would have produced.</li>
|
||||
</ul>
|
||||
<h3 id="phase-4-column-major-matrix-fetch-roadmap-points-78-implemented-numa-parallelism-deferred">Phase 4 — Column-major matrix fetch (roadmap points 7–8) — implemented, NUMA parallelism deferred</h3>
|
||||
<p><strong>Goal (revised during implementation)</strong>: replace <code>fill_row</code>-per-hit (row-major, worst-case mmap locality) with a column-major scan. <code>PartitionRunner</code> turned out to be the wrong mechanism for this at this call granularity — see below; the column-major fetch itself is implemented and validated, without it.</p>
|
||||
<p><strong>What shipped:</strong>
|
||||
- <code>obicompactvec</code>: the per-column accessors this phase needed <strong>already existed</strong> — <code>PersistentCompactIntMatrix::col_view(c)</code> and <code>PersistentBitMatrix::col_view(c)</code> are public, and <code>IntSliceView::get(slot)</code>/<code>BitSliceView::get(slot)</code> are public — the original plan underestimated how much of this plumbing the pairwise-distance code (<code>dump</code>/<code>select</code>/<code>stats</code>) had already required. The one real gap: <code>PersistentBitMatrix::col_view()</code> panics on the <code>Implicit</code> variant (the documented mono-genome fast path, <code>bitmatrix.rs</code>). Added <code>PersistentBitMatrix::get(c, slot) -> u32</code> (<code>bitmatrix.rs</code>), a non-panicking column-major point lookup that returns <code>1</code> for <code>Implicit</code> regardless of <code>c</code> — the smallest surface needed, not a new <code>col_get</code> API from scratch.
|
||||
- <code>obikpartitionner/src/query_layer.rs</code>: <code>query_partition_with</code> is now two explicit stages, matching roadmap points 6–8: <strong>stage 1</strong> (MPHF-only, per unique k-mer, bucket hits by <code>(layer_idx, slot)</code>, emits <code>QueryHit::Found</code>) then <strong>stage 2</strong> (per layer with ≥1 hit, column-major: for each genome column <code>g</code> in <code>0..layer.n_cols().min(n_genomes)</code>, scan that layer's bucketed slots and call <code>col_value(g, slot)</code>, emitting <code>QueryHit::Value(descs, g, value)</code> on nonzero). <code>QueryHit</code> is a single enum delivered through one <code>FnMut(QueryHit)</code> callback — an earlier two-closure design (<code>on_found</code> + <code>on_value</code>) didn't borrow-check, since the caller's single mutable accumulator (<code>KmerResults</code>) can't be captured by two separate <code>FnMut</code> closures passed to the same call.
|
||||
- <code>obikindex/src/partition/query_layer.rs</code>: <code>query_partition_with</code> is now two explicit stages, matching roadmap points 6–8: <strong>stage 1</strong> (MPHF-only, per unique k-mer, bucket hits by <code>(layer_idx, slot)</code>, emits <code>QueryHit::Found</code>) then <strong>stage 2</strong> (per layer with ≥1 hit, column-major: for each genome column <code>g</code> in <code>0..layer.n_cols().min(n_genomes)</code>, scan that layer's bucketed slots and call <code>col_value(g, slot)</code>, emitting <code>QueryHit::Value(descs, g, value)</code> on nonzero). <code>QueryHit</code> is a single enum delivered through one <code>FnMut(QueryHit)</code> callback — an earlier two-closure design (<code>on_found</code> + <code>on_value</code>) didn't borrow-check, since the caller's single mutable accumulator (<code>KmerResults</code>) can't be captured by two separate <code>FnMut</code> closures passed to the same call.
|
||||
- <code>obikmer/src/cmd/query.rs</code>: <code>KmerResults::set</code> (row-major, whole-row-at-once) replaced by <code>mark_found</code> (stage 1: flag a position as indexed, independent of any genome's value) and <code>set_one</code> (stage 2: write one genome's value at one position). <code>QueryStats</code> extended with <code>n_columns_scanned</code>/<code>n_col_get_calls</code>, logged per chunk.
|
||||
- Total <code>get()</code>-equivalent calls are unchanged from the row-major version (<code>n_hits × n_cols</code> in the worst case, confirmed by <code>n_col_get_calls</code> in the debug log) — the win is locality (sequential access within one layer's column at a time, across <code>mmap</code>'d regions, instead of jumping across all columns per hit), exactly as predicted.</p>
|
||||
<p><strong>What did not ship, and why — <code>PartitionRunner</code> is architecturally the wrong tool here:</strong>
|
||||
Reading <code>obikindex/src/numa.rs</code>'s actual <code>run()</code> body (not just its doc comments) shows every call spawns a timer thread <strong>plus one OS thread per worker slot on every NUMA node</strong> (<code>std::thread::scope</code> + one <code>s.spawn()</code> per node per <code>max_workers</code>) — on the 192-core/8-NUMA reference machine, that's on the order of 190+ fresh OS threads spawned <strong>per call</strong>. This is fine for its actual, established usage in this codebase (<code>merge.rs</code>, <code>index.rs</code>'s <code>build_layers</code>): one <code>PartitionRunner::new()</code> + one <code>run()</code> call per command invocation, amortised over a batch of ~256 long-running partitions. It is not fine for <code>query</code>'s call pattern: <code>query_partition_with</code> runs once per <code>(chunk, partition)</code>, potentially thousands of times per second — spawning ~190 OS threads that often to scan a handful of genome columns would very likely cost far more than the row-major approach it's meant to replace. This is exactly the "resolve empirically, don't assume" composition risk the roadmap flagged, just resolved by reading the mechanism's actual cost before wiring it in, rather than by measuring a regression on the cluster after the fact.
|
||||
The column-major loop in stage 2 is therefore a <strong>plain sequential loop</strong> for now — it captures the whole, provable locality win (roadmap point 8's actual claim) without adding any parallelism mechanism. Genome-column-level parallelism (point 8's "bonus" axis) and partition-level parallelism (point 7) are both deferred — not abandoned. Candidates for a follow-up, once there's a concrete profiling need: (a) <code>rayon</code>'s already-warm global pool (<code>into_par_iter()</code>) for the column axis specifically — cheap to invoke repeatedly since it doesn't spawn threads per call, though it's the same "naive rayon" pattern <code>numa_worker_pools.md</code> warns about for a <em>different</em> workload (random pointer-chasing over large hash maps); a column scan's access pattern (sequential reads within one <code>mmap</code>'d region) has a different contention profile and hasn't been shown to have the same problem — needs its own measurement, not an assumption either way; (b) restructuring so <code>PartitionRunner</code> is invoked once per whole <code>query</code> run (or per large batch of chunks) rather than per <code>(chunk, partition)</code>, amortising its spawn cost the way <code>merge</code>/<code>build_layers</code> do — a bigger structural change than this phase's scope.
|
||||
- Log (implemented): <code>QueryStats::n_columns_scanned</code>/<code>n_col_get_calls</code>, folded into the existing per-chunk <code>debug!("k-mer dedup + column-major fetch", ...)</code> line (<code>query.rs</code>) alongside phase 3's dedup counters.
|
||||
- <strong>Unit tests</strong>: extended <code>obikpartitionner/src/tests/query_layer.rs</code> (phase 3's file) — <code>query_partition_with</code>'s empty/missing-index paths updated for the new <code>QueryStats</code> fields and single-callback signature.
|
||||
- <strong>Unit tests</strong>: extended <code>obikindex/src/partition/tests/query_layer.rs</code> (phase 3's file) — <code>query_partition_with</code>'s empty/missing-index paths updated for the new <code>QueryStats</code> fields and single-callback signature.
|
||||
- <strong>Validation performed</strong>: full workspace build + <code>cargo test --workspace</code>, zero failures. Functional validation against real indexes: (1) a single-genome index — output byte-identical to pre-phase-4 (same <code>kmer_count</code>/<code>kmer_strict_matches</code> on every record); (2) the existing 20-genome <code>benchmark/global_index_presence</code> index — runs correctly, <code>n_hits=0</code> for an unrelated query (expected: no shared k-mers between a plant read and a bacterial reference set), no panics, confirming the <code>Implicit</code>/multi-column bounds logic doesn't crash on a real multi-genome, mixed-format index; (3) <strong>the critical correctness case</strong>: built two single-sequence-pair test genomes, merged into one 2-genome index, queried with reads from both — reads from <code>genomeA</code> matched <strong>only</strong> <code>genomeA</code> (<code>kmer_count</code> identical to the pre-dedup occurrence count, zero leakage into <code>genomeB</code>'s column) and vice versa. This is the test that would have caught a column-index mixup, an off-by-one in <code>n_cols</code>, or cross-genome bleed from the stage-1/stage-2 split — it passed cleanly.
|
||||
- <strong>Not yet done</strong>: the microbenchmark comparing column-major vs. the old row-major access pattern's wall time / page-fault counters on a large-<code>n_genomes</code> layer — needs a realistically large multi-genome index and, for the page-fault counters specifically, Linux (not available from this development environment). Left for cluster validation alongside phases 1–3's own pending measurements.</p>
|
||||
<h3 id="phase-5-sparse-findere-rework-roadmap-point-9">Phase 5 — Sparse Findere rework (roadmap point 9)</h3>
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1417,9 +1445,9 @@ IO pattern: sequential read of each source column → random write into new colu
|
||||
</table>
|
||||
<h2 id="files-to-modify">Files to modify</h2>
|
||||
<ul>
|
||||
<li><code>src/obikpartitionner/src/rebuild_layer.rs</code> — <code>rebuild_partition</code> and <code>iter_src_layers</code></li>
|
||||
<li><code>src/obikindex/src/partition/rebuild_layer.rs</code> — <code>rebuild_partition</code> and <code>iter_src_layers</code></li>
|
||||
<li>Possibly <code>src/obicompactvec/</code> — add column iterator API if not already present</li>
|
||||
<li><code>src/obilayeredmap/</code> — check if per-column sequential access is exposed on <code>SrcLayerData</code></li>
|
||||
<li><code>src/obikindex/src/layer/</code> — check if per-column sequential access is exposed on <code>SrcLayerData</code></li>
|
||||
</ul>
|
||||
<h2 id="open-questions">Open questions</h2>
|
||||
<ul>
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
|
||||
|
||||
<link rel="prev" href="../../../implementation/benchmark_query_testing/">
|
||||
<link rel="prev" href="../../../implementation/partition_layer_cache/">
|
||||
|
||||
|
||||
<link rel="next" href="../../index_architecture/">
|
||||
@@ -731,14 +731,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -976,6 +976,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -731,14 +731,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -976,6 +976,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1328,6 +1356,17 @@
|
||||
</span>
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#implemented-2026-08-20" class="md-nav__link">
|
||||
<span class="md-ellipsis">
|
||||
|
||||
Implemented (2026-08-20)
|
||||
|
||||
</span>
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
@@ -1602,6 +1641,17 @@
|
||||
</span>
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#implemented-2026-08-20" class="md-nav__link">
|
||||
<span class="md-ellipsis">
|
||||
|
||||
Implemented (2026-08-20)
|
||||
|
||||
</span>
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
@@ -1637,7 +1687,7 @@ one-way function, not an invertible bijection with a stored inverse. Any
|
||||
method that reconstructs a kmer from a bare slot number is wrong by
|
||||
construction, regardless of the mechanism used (MPHF re-hash, or evidence
|
||||
decode + direct unitig read). See <code>MphfLayer::kmer_at</code>
|
||||
(<code>obilayeredmap/src/mphf_layer.rs</code>) — flagged for removal, currently called
|
||||
(<code>obikindex/src/layer/mphf_layer.rs</code>) — flagged for removal, currently called
|
||||
from <code>obikphylo/siblings/build.rs</code> and <code>family_scan.rs</code> (since removed — see
|
||||
"Pending work" status below).</p>
|
||||
<h2 id="two-pipelines-never-mixed">Two pipelines, never mixed</h2>
|
||||
@@ -1700,7 +1750,7 @@ partition is unknown) and must keep going through
|
||||
<code>index()</code>.</p>
|
||||
<h2 id="pending-work-done">Pending work — done</h2>
|
||||
<p>The plan above shipped: <code>obikphylo</code> (a new crate — phylo-domain extension
|
||||
traits over <code>obikindex::KmerIndex</code>/<code>obilayeredmap::Layer<D></code>, replacing the
|
||||
traits over <code>obikindex::KmerIndex</code>/<code>obikindex::layer::Layer<D></code>, replacing the
|
||||
old <code>obikindex::siblings</code> module) builds and reads the annex purely in
|
||||
iteration order (<code>SiblingLayerExt::iter_siblings</code>/<code>iter_minorants</code>, both with
|
||||
batch variants, mirroring <code>Layer<D></code>'s own <code>KmerIter</code>/<code>KmerBatchIter</code>
|
||||
@@ -2268,13 +2318,13 @@ BRWT-style column-correlation exploitation.</p>
|
||||
<p><code>PersistentSparseBitMatrix</code> went from a validated but unused type to a
|
||||
real, selectable on-disk format:</p>
|
||||
<ul>
|
||||
<li><strong>Generic <code>Layer<D></code></strong>: <code>obilayeredmap::Layer<D></code>'s presence-only methods
|
||||
<li><strong>Generic <code>Layer<D></code></strong>: <code>obikindex::layer::Layer<D></code>'s presence-only methods
|
||||
(<code>n_cols</code>, <code>sub_matrix</code>, <code>fill_sub_matrix</code>) are generic over any
|
||||
<code>D: LayerData<Item = Box<[bool]>> + BinaryMatrix</code>, not hardcoded to
|
||||
<code>PersistentBitMatrix</code> — <code>PersistentSparseBitMatrix</code> implements
|
||||
<code>LayerData</code> (<code>open</code>/<code>read</code>) the same way. <code>find_slot</code>/<code>index_batch</code> were
|
||||
already generic over any <code>D: LayerData</code>, so they needed no change.
|
||||
Verified by <code>obilayeredmap</code>'s
|
||||
Verified by <code>obikindex::layer</code>'s
|
||||
<code>presence_layer_generic_over_sparse_matches_dense</code> test: build a dense
|
||||
presence layer, convert it to sparse via <code>build_from_dense</code>, open both
|
||||
as <code>Layer<PersistentBitMatrix></code>/<code>Layer<PersistentSparseBitMatrix></code> on
|
||||
@@ -2284,7 +2334,7 @@ real, selectable on-disk format:</p>
|
||||
<code>AtomicUsize</code>s in test builds, not thread-local, so a test using a
|
||||
different <code>k</code> races every other test in the same crate binary; a k=11
|
||||
version of this test passed alone but failed under the full
|
||||
<code>obilayeredmap</code> suite for exactly that reason before being fixed.)</li>
|
||||
<code>obikindex::layer</code> suite for exactly that reason before being fixed.)</li>
|
||||
<li><strong><code>obikphylo::siblings::cache::Mat</code></strong> gained a third variant,
|
||||
<code>SparsePresence(Layer<PersistentSparseBitMatrix>)</code>, alongside <code>Count</code>
|
||||
and <code>Presence</code> — every method (<code>find_slot</code>, <code>index_batch</code>,
|
||||
@@ -2314,7 +2364,7 @@ real, selectable on-disk format:</p>
|
||||
dense-path test (<code>sibling_annex_one_sibling_each</code>) exactly — proves the
|
||||
sparse format round-trips through the real build pipeline
|
||||
(<code>PartitionCache</code> sparse-detection included), not just the
|
||||
<code>obicompactvec</code>/<code>obilayeredmap</code> unit layers below it.</li>
|
||||
<code>obicompactvec</code>/<code>obikindex::layer</code> unit layers below it.</li>
|
||||
</ul>
|
||||
<p>Full workspace <code>cargo test</code> (all crates, unit + doc tests) green after
|
||||
this change.</p>
|
||||
@@ -2360,7 +2410,7 @@ this change.</p>
|
||||
(activation: either given; defaults <code>1.0</code>/<code>0.5</code> for whichever is unset).</li>
|
||||
</ul>
|
||||
<p>Full workspace <code>cargo test</code> green after this change (167 unit tests in
|
||||
<code>obicompactvec</code>+<code>obilayeredmap</code>+<code>obikphylo</code> alone, plus every other
|
||||
<code>obicompactvec</code>+<code>obikindex::layer</code>+<code>obikphylo</code> alone, plus every other
|
||||
crate's suite, no regressions).</p>
|
||||
<p><strong>Still open, not part of this change</strong> (per "Correction to the 'single
|
||||
pass' framing" above): <code>--raw-snp-distance</code>/<code>--raw-snp-counts</code> (the
|
||||
@@ -2380,7 +2430,7 @@ reproducible across two runs with warm disk cache), the opposite of
|
||||
<code>pack --sparse</code>'s stated intent ("faster for single-row access... like
|
||||
query").</p>
|
||||
<p><strong>Root cause, read from source, not measured in isolation:</strong>
|
||||
<code>KmerPartition::query_partition_with</code> (<code>obikpartitionner/src/query_layer.rs:155-220</code>)
|
||||
<code>KmerPartition::query_partition_with</code> (<code>obikindex/src/partition/query_layer.rs:155-220</code>)
|
||||
is architecturally column-major: stage 2 walks <code>for g in 0..n_cols { for
|
||||
slot in hit_slots { layer.col_value(g, slot) } }</code>, documented (correctly)
|
||||
as the right locality strategy for the packed/columnar formats, where
|
||||
@@ -2407,45 +2457,131 @@ consumes as <code>QueryHit::Value</code> — not a materialized sub-matrix.</p>
|
||||
<p><strong>Proposed primitive</strong> (design only, not implemented — explicit ask: keep
|
||||
count matrices <em>not excluded</em>, even though effort right now is
|
||||
presence/absence only):</p>
|
||||
<div class="highlight"><pre><span></span><code><span class="sd">/// Visit every nonzero cell among `slots`. Order unspecified.</span>
|
||||
<span class="k">fn</span><span class="w"> </span><span class="nf">for_each_nonzero</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">slots</span><span class="p">:</span><span class="w"> </span><span class="kp">&</span><span class="p">[</span><span class="kt">usize</span><span class="p">],</span><span class="w"> </span><span class="n">f</span><span class="p">:</span><span class="w"> </span><span class="nc">impl</span><span class="w"> </span><span class="nb">FnMut</span><span class="p">(</span><span class="kt">usize</span><span class="w"> </span><span class="cm">/*idx into slots*/</span><span class="p">,</span><span class="w"> </span><span class="kt">usize</span><span class="w"> </span><span class="cm">/*col*/</span><span class="p">,</span><span class="w"> </span><span class="kt">u32</span><span class="w"> </span><span class="cm">/*value*/</span><span class="p">));</span>
|
||||
<p>Not a closure-driven <code>for_each</code> — a real <code>Iterator</code>, one concrete struct
|
||||
per matrix format, so the traversal state (current position in the sorted
|
||||
slot list, current column, permutation, sparse-row decode cursor…) lives
|
||||
in named struct fields instead of being threaded implicitly through
|
||||
recursion or a captured closure. RPITIT (stable since 1.75, and this
|
||||
workspace is edition 2024) means the trait method can return it without
|
||||
naming or boxing the concrete type:</p>
|
||||
<div class="highlight"><pre><span></span><code><span class="sd">/// Yields every nonzero cell among `slots`, in implementation-defined order.</span>
|
||||
<span class="k">fn</span><span class="w"> </span><span class="nf">nonzero_iter</span><span class="o"><'</span><span class="na">a</span><span class="o">></span><span class="p">(</span><span class="o">&'</span><span class="na">a</span><span class="w"> </span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">slots</span><span class="p">:</span><span class="w"> </span><span class="kp">&</span><span class="o">'</span><span class="na">a</span><span class="w"> </span><span class="p">[</span><span class="kt">usize</span><span class="p">])</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">impl</span><span class="w"> </span><span class="nb">Iterator</span><span class="o"><</span><span class="n">Item</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="kt">usize</span><span class="p">,</span><span class="w"> </span><span class="kt">usize</span><span class="p">,</span><span class="w"> </span><span class="kt">u32</span><span class="p">)</span><span class="o">></span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="o">'</span><span class="na">a</span><span class="p">;</span>
|
||||
<span class="c1">// item: (idx into `slots`, col, value)</span>
|
||||
</code></pre></div>
|
||||
<p>This is the one foundational primitive per format — both <code>fill_sub_matrix</code>
|
||||
and a <code>for_each</code>-style callback become trivial, free consumers of it
|
||||
(<code>.for_each(f)</code> is already <code>Iterator::for_each</code> from std; <code>fill_sub_matrix</code>
|
||||
becomes "drain the iterator, scatter into <code>out[][]</code>"), instead of two
|
||||
independently-maintained traversals that can silently diverge (see the bug
|
||||
below — this is exactly how it happened).</p>
|
||||
<ul>
|
||||
<li>On <code>PersistentSparseBitMatrix</code>: native override, one pass per slot via
|
||||
the existing (currently private) <code>for_each_genome_in_row</code> — O(Σ row
|
||||
nnz), zero n_cols-wide allocation. This is the whole point: expose code
|
||||
that already exists rather than write anything new for the sparse side.</li>
|
||||
<li>On <code>PersistentBitMatrix::{Packed,Columnar,Implicit}</code>: <strong>provided
|
||||
default</strong>, derived from <code>fill_sub_matrix</code> (materialize, then filter to
|
||||
true cells) — reuses the already-optimal column-major/mmap path for
|
||||
those formats, no new code needed there either.</li>
|
||||
<li><strong>On <code>PersistentCompactIntMatrix</code> (counts)</strong>: same provided-default
|
||||
treatment, derived from its own existing <code>fill_sub_matrix</code> (<code>u32</code>-typed
|
||||
already, <code>intmatrix.rs:387</code>) — not hand-optimized (no sparse count
|
||||
format exists — "Explicitly deferred" per <code>traits.rs:9-12</code>), but not
|
||||
excluded either: it gets a working, not-pathological implementation for
|
||||
free today, on the same trait, ready for a native override the day a
|
||||
sparse count format lands. This is why the signature carries <code>u32</code>
|
||||
rather than <code>bool</code> — presence is <code>0</code>/<code>1</code>, counts are <code>u32</code>, one trait
|
||||
covers both without a bool/u32 split forcing counts out of the design.</li>
|
||||
<li><strong>On <code>PersistentSparseBitMatrix</code></strong>: the struct is nearly free to write —
|
||||
it wraps the existing (currently private) <code>for_each_genome_in_row</code>
|
||||
per-row decode, advancing to the next <code>slots</code> entry on exhaustion. O(Σ
|
||||
row nnz), zero <code>n_cols</code>-wide allocation.</li>
|
||||
<li><strong>On <code>PersistentBitMatrix::{Packed,Columnar}</code></strong>: revised — cheaper than
|
||||
first thought, by reusing the same split already used for
|
||||
<code>fill_matrix</code>'s own implementation instead of hand-writing a resumable
|
||||
state machine at the matrix level. The base-vector layer
|
||||
(<code>BitSliceView</code>, <code>views.rs</code>) already separates the two concerns:
|
||||
<code>fill_batch_sorted</code> (<code>views.rs:55-60</code>, sorted-slot batch lookup) and a
|
||||
genuine per-bit <code>Iterator</code> (<code>BitSliceIter</code>, <code>views.rs:94+</code>) sit side by
|
||||
side there, one level below the matrix. Adding a
|
||||
"positions among <code>sorted_slots</code> where the bit is set" iterator at that
|
||||
same vector level is a <code>filter</code> over the existing <code>get()</code> — no new
|
||||
state machine, since <code>std::iter::Filter</code> already <em>is</em> one, generated by
|
||||
the compiler:
|
||||
<div class="highlight"><pre><span></span><code><span class="k">fn</span><span class="w"> </span><span class="nf">nonzero_among_sorted</span><span class="o"><'</span><span class="na">s</span><span class="o">></span><span class="p">(</span><span class="o">&'</span><span class="na">s</span><span class="w"> </span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">sorted_slots</span><span class="p">:</span><span class="w"> </span><span class="kp">&</span><span class="o">'</span><span class="na">s</span><span class="w"> </span><span class="p">[</span><span class="kt">usize</span><span class="p">])</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">impl</span><span class="w"> </span><span class="nb">Iterator</span><span class="o"><</span><span class="n">Item</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="kt">usize</span><span class="o">></span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="o">'</span><span class="na">s</span><span class="w"> </span><span class="p">{</span>
|
||||
<span class="w"> </span><span class="n">sorted_slots</span><span class="p">.</span><span class="n">iter</span><span class="p">().</span><span class="n">copied</span><span class="p">().</span><span class="n">filter</span><span class="p">(</span><span class="k">move</span><span class="w"> </span><span class="o">|&</span><span class="n">slot</span><span class="o">|</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">get</span><span class="p">(</span><span class="n">slot</span><span class="p">))</span>
|
||||
<span class="p">}</span>
|
||||
</code></pre></div>
|
||||
The matrix-level <code>nonzero_iter</code> then composes these per column with
|
||||
<code>flat_map</code> over <code>0..n_cols</code> (each column's hits, tagged with <code>c</code>,
|
||||
slot mapped back through the sort permutation <code>fill_batch</code>/
|
||||
<code>fill_batch_sorted</code> already carry) — again a combinator chain, not a
|
||||
hand-rolled struct. Same algorithm, same mmap/sort locality as today's
|
||||
<code>fill_sub_matrix</code>; just assembled from <code>std</code> iterator adaptors instead
|
||||
of a loop body writing into a buffer, mirroring the vector/matrix split
|
||||
the codebase already uses for <code>fill_batch_sorted</code> rather than
|
||||
introducing a new shape.</li>
|
||||
<li><strong><code>Implicit</code></strong>: trivial (<code>slots.iter().map(|&i| (i, 0, 1))</code>, one column,
|
||||
always present).</li>
|
||||
<li><strong>On <code>PersistentCompactIntMatrix</code> (counts)</strong>: same treatment as
|
||||
<code>Packed</code>/<code>Columnar</code> — no sparse count format exists yet ("Explicitly
|
||||
deferred" per <code>traits.rs:9-12</code>), so no native low-effort case the way
|
||||
<code>Sparse</code> has one, but not excluded either: the iterator's <code>Item</code> is
|
||||
already <code>(usize, usize, u32)</code>, not <code>bool</code>, specifically so presence
|
||||
(<code>0</code>/<code>1</code>) and counts (arbitrary <code>u32</code>) share one primitive instead of a
|
||||
bool/u32 split forcing counts out of the design. Ready for a native
|
||||
sparse-count struct later without a signature change.</li>
|
||||
</ul>
|
||||
<p>Would let <code>query_partition_with</code>'s stage 2 collapse to one
|
||||
<code>layer.matrix().for_each_nonzero(&hit_slots, |i, g, v| on_event(...))</code>
|
||||
call per layer, format-agnostic, with each backend's existing (or
|
||||
default-derived) implementation deciding the actual access pattern.</p>
|
||||
<p><strong>Separately, an existing bug in the generic path</strong> (found while tracing
|
||||
this, itself not yet fixed): <code>PersistentBitMatrix::fill_sub_matrix</code>
|
||||
(<code>persistent.rs:190-215</code>, the enum wrapper backing <code>BinaryMatrix</code>'s
|
||||
default trait impl) does <em>not</em> delegate to
|
||||
<code>for (i, g, v) in layer.matrix().nonzero_iter(&hit_slots) { on_event(...) }</code>
|
||||
per layer, format-agnostic, each backend's struct deciding the actual
|
||||
traversal.</p>
|
||||
<p><strong>This also closes the existing dispatch bug for free, by construction</strong>:
|
||||
<code>PersistentBitMatrix::fill_sub_matrix</code> (<code>persistent.rs:190-215</code>, the enum
|
||||
wrapper backing <code>BinaryMatrix</code>'s trait impl) today does <em>not</em> delegate to
|
||||
<code>PersistentSparseBitMatrix::fill_sub_matrix</code> for <code>Self::Sparse</code> — it
|
||||
reimplements the same naive per-(column, slot) <code>fill_row_bool</code> loop
|
||||
instead, bypassing the efficient native method one file over
|
||||
(<code>sparse.rs:249-258</code>). <code>obikphylo::siblings::cache::Mat</code>
|
||||
(<code>cache.rs:138-145</code>) independently built its own parallel enum wrapper
|
||||
that dispatches correctly — a sign this was worked around rather than
|
||||
fixed at the source. Any future <code>for_each_nonzero</code> work should fix this
|
||||
dispatch too (or route through it), rather than adding a third
|
||||
independently-dispatching wrapper.</p>
|
||||
that dispatches correctly — evidence this was worked around rather than
|
||||
fixed at the source: two hand-written traversals for the same format,
|
||||
free to drift apart, and they did. If <code>fill_sub_matrix</code> itself is
|
||||
rewritten as "drain <code>nonzero_iter</code>, scatter into <code>out[][]</code>", there is only
|
||||
one traversal per format left to get right — the bug class doesn't just
|
||||
get fixed once, it stops being possible to reintroduce.</p>
|
||||
<h2 id="implemented-2026-08-20">Implemented (2026-08-20)</h2>
|
||||
<p>Built as designed above, with one deviation from the original sketch:
|
||||
<code>nonzero_iter</code> ended up <code>Box<dyn Iterator<...>></code>, not a bare <code>impl
|
||||
Iterator</code>, because <code>Columnar</code>/<code>Packed</code>/<code>Sparse</code>/<code>Implicit</code> are genuinely
|
||||
different concrete types and this method isn't on a trait (kept off
|
||||
<code>BinaryMatrix</code> deliberately — that trait is used as <code>dyn BinaryMatrix</code> in
|
||||
<code>tests/sparse.rs</code>, and RPITIT methods aren't dyn-compatible). One <code>Box</code>
|
||||
per <code>nonzero_iter</code> call, not per cell — negligible next to what it
|
||||
replaces.</p>
|
||||
<ul>
|
||||
<li><code>BitSliceView::nonzero_among_sorted</code> / <code>IntSliceView::nonzero_among_sorted</code>
|
||||
(<code>obicompactvec/src/views.rs</code>): the vector-level <code>filter</code>/<code>filter_map</code>
|
||||
primitive, exactly as sketched — no new state machine, <code>std</code>'s own.</li>
|
||||
<li><code>PersistentSparseBitMatrix::nonzero_iter</code> (<code>bitmatrix/sparse.rs</code>): native,
|
||||
<code>std::iter::from_fn</code> over one buffered row at a time via the existing
|
||||
<code>for_each_genome_in_row</code> — no <code>n_cols</code>-wide allocation, ever.</li>
|
||||
<li><code>PersistentBitMatrix::nonzero_iter</code> (<code>bitmatrix/persistent.rs</code>): dispatches
|
||||
to the above for <code>Sparse</code>; for <code>Columnar</code>/<code>Packed</code>, loops columns,
|
||||
collects each column's <code>nonzero_among_sorted</code> hits via <code>.extend()</code> (not
|
||||
<code>flat_map</code> — a <code>flat_map</code> closure can't lazily return something
|
||||
borrowing its own captured sort permutation across separate calls
|
||||
without either boxing per-column or fighting the borrow checker; eager
|
||||
collection into one <code>Vec</code> sidesteps it, at zero cost since
|
||||
<code>fill_sub_matrix</code> already fully materialized anyway). <code>Implicit</code> trivial.</li>
|
||||
<li><code>PersistentBitMatrix::fill_sub_matrix</code> and <code>sub_matrix</code> rewritten to
|
||||
drain <code>nonzero_iter</code> — the dispatch bug is gone because there is now
|
||||
only one traversal per format, not because the old one was patched.
|
||||
<code>PersistentCompactIntMatrix::nonzero_iter</code> added the same way (counts
|
||||
not excluded, per the earlier ask) — no native low-effort case, since no
|
||||
sparse count format exists, but on the same primitive, ready for one.</li>
|
||||
<li><code>KmerPartition::query_partition_with</code> (<code>obikindex/src/partition/query_layer.rs</code>):
|
||||
stage 2's column-major <code>for g { for slot { col_value } }</code> replaced by one
|
||||
<code>layer.nonzero_iter(&slot_list)</code> call per layer, format-agnostic.</li>
|
||||
<li>Tests: <code>nonzero_iter_matches_dense</code>, <code>nonzero_iter_matches_row</code>, and —
|
||||
the one that actually targets the dispatch bug rather than each type's
|
||||
own correctness — <code>enum_wrapper_dispatches_to_native_sparse</code> (builds
|
||||
<code>PersistentBitMatrix::Sparse(...)</code> directly, not through <code>open</code>, since
|
||||
<code>open</code> only auto-detects <code>Sparse</code> from a <code>presence/</code> dir layout).
|
||||
<code>cargo test --workspace</code>: green, no regressions.</li>
|
||||
</ul>
|
||||
<p><strong>Measured</strong>: re-ran the <code>benchmark/</code> query branch (100k reads × 2
|
||||
specimens, same setup as the original finding). Correctness still 0
|
||||
mismatches. The dense/sparse performance gap is gone — previously sparse
|
||||
~30-50% slower than dense, reproducibly; now within ~1-3% either way
|
||||
(7.42s dense vs 7.60s sparse for <code>Escherichia_coli--K-12_MG1655</code>; 5.25s vs
|
||||
5.30s for <code>Saccharolobus_islandicus--M.16.4</code>) — noise-level, not a
|
||||
systematic gap. <code>pack --sparse</code>'s claimed query win isn't confirmed
|
||||
outright by this (sparse should arguably now <em>beat</em> dense on truly sparse
|
||||
real data, not just tie), but the pathological regression is fixed.</p>
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<link rel="prev" href="../obitaxonomy/">
|
||||
|
||||
|
||||
<link rel="next" href="../../architecture/sequences/invariant/">
|
||||
<link rel="next" href="../partition_layer_cache/">
|
||||
|
||||
|
||||
|
||||
@@ -733,14 +733,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1113,6 +1113,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -405,11 +405,11 @@
|
||||
</a>
|
||||
</li>
|
||||
<li class="md-nav__item">
|
||||
<a class="md-nav__link" href="../obilayeredmap/">
|
||||
<a class="md-nav__link" href="../layer_tier/">
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -509,6 +509,18 @@
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="md-nav__item">
|
||||
<a class="md-nav__link" href="../partition_layer_cache/">
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
</a>
|
||||
</li>
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1242,9 +1270,9 @@
|
||||
<h1 id="coverage-implementationevidence_eliminationmd">Coverage: implementation/evidence_elimination.md</h1>
|
||||
<h2 id="code-couvert">Code couvert</h2>
|
||||
<ul>
|
||||
<li><code>obilayeredmap/src/fingerprint.rs</code> — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches()</li>
|
||||
<li><code>obilayeredmap/src/mphf_layer.rs</code> — build_approx_evidence(dir, b, z), find_approx()</li>
|
||||
<li><code>obilayeredmap/src/meta.rs</code> — EvidenceKind::Approx { b, z }, LayerMeta</li>
|
||||
<li><code>obikindex/src/layer/fingerprint.rs</code> — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches()</li>
|
||||
<li><code>obikindex/src/layer/mphf_layer.rs</code> — build_approx_evidence(dir, b, z), find_approx()</li>
|
||||
<li><code>obikindex/src/layer/meta.rs</code> — EvidenceKind::Approx { b, z }, LayerMeta</li>
|
||||
<li><code>obikindex/src/reindex.rs</code> — KmerIndex::reindex(), conversion exact↔approx en place</li>
|
||||
<li><code>obikmer/src/cmd/reindex.rs</code> — CLI reindex, options --approx, -z, --evidence-bits, --fp, --block-size</li>
|
||||
<li><code>obikmer/src/cmd/index.rs</code> — resolve_approx_params(), options --approx, -z, --evidence-bits, --fp</li>
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<link rel="prev" href="../unitig_evidence/">
|
||||
|
||||
|
||||
<link rel="next" href="../obilayeredmap/">
|
||||
<link rel="next" href="../layer_tier/">
|
||||
|
||||
|
||||
|
||||
@@ -868,14 +868,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1113,6 +1113,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -733,14 +733,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1236,6 +1236,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -2158,7 +2186,7 @@ obikmer<span class="w"> </span>phylo<span class="w"> </span>myindex<span class="
|
||||
<h2 id="implementation">Implementation</h2>
|
||||
<ul>
|
||||
<li>
|
||||
<p><strong><code>obikpartitionner::filter::GroupQuorumFilter</code></strong> — implements <code>KmerFilter</code>
|
||||
<p><strong><code>obikindex::partition::filter::GroupQuorumFilter</code></strong> — implements <code>KmerFilter</code>
|
||||
using pre-computed ingroup and outgroup index vectors. The heavy logic
|
||||
(predicate parsing, three-value evaluation, genome classification) happens
|
||||
once before any iteration; each k-mer row evaluation is a simple index
|
||||
@@ -2171,7 +2199,7 @@ obikmer<span class="w"> </span>phylo<span class="w"> </span>myindex<span class="
|
||||
list.</p>
|
||||
</li>
|
||||
<li>
|
||||
<p><strong><code>obikpartitionner::KmerPartition::iter_partition_kmers</code></strong> — accepts
|
||||
<p><strong><code>obikindex::partition::KmerPartition::iter_partition_kmers</code></strong> — accepts
|
||||
<code>filters: &[Box<dyn KmerFilter>]</code> and applies them per-kmer before invoking
|
||||
the callback. <code>filter</code>, <code>dump</code>, and <code>unitig</code> all go through this single
|
||||
entry point.</p>
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -868,14 +868,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1113,6 +1113,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
+40
-12
@@ -19,7 +19,7 @@
|
||||
|
||||
|
||||
|
||||
<title>Obilayeredmap.refs - obikmer</title>
|
||||
<title>Layer tier.refs - obikmer</title>
|
||||
|
||||
|
||||
|
||||
@@ -60,7 +60,7 @@
|
||||
<div data-md-component="skip">
|
||||
|
||||
|
||||
<a href="#coverage-implementationobilayeredmapmd" class="md-skip">
|
||||
<a href="#coverage-implementationlayer_tiermd" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
@@ -96,7 +96,7 @@
|
||||
<div class="md-header__topic" data-md-component="header-topic">
|
||||
<span class="md-ellipsis">
|
||||
|
||||
Obilayeredmap.refs
|
||||
Layer tier.refs
|
||||
|
||||
</span>
|
||||
</div>
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1239,15 +1267,15 @@
|
||||
|
||||
|
||||
<!-- coverage sidecar — ne pas ajouter au nav mkdocs -->
|
||||
<h1 id="coverage-implementationobilayeredmapmd">Coverage: implementation/obilayeredmap.md</h1>
|
||||
<h1 id="coverage-implementationlayer_tiermd">Coverage: implementation/layer_tier.md</h1>
|
||||
<h2 id="code-couvert">Code couvert</h2>
|
||||
<ul>
|
||||
<li><code>obilayeredmap/src/mphf_layer.rs</code> — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx()</li>
|
||||
<li><code>obilayeredmap/src/layer.rs</code> — Layer<D>, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column()</li>
|
||||
<li><code>obilayeredmap/src/map.rs</code> — LayeredMap<D>, push_layer(), query()</li>
|
||||
<li><code>obilayeredmap/src/evidence.rs</code> — Evidence, EvidenceWriter, encodage chunk_id:rank</li>
|
||||
<li><code>obilayeredmap/src/fingerprint.rs</code> — FingerprintVec, FingerprintVecWriter, matches()</li>
|
||||
<li><code>obilayeredmap/src/meta.rs</code> — LayerMeta, EvidenceKind (Exact / Approx { b, z })</li>
|
||||
<li><code>obikindex/src/layer/mphf_layer.rs</code> — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx()</li>
|
||||
<li><code>obikindex/src/layer/typed_layer.rs</code> — Layer<D>, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column()</li>
|
||||
<li><code>obikindex/src/layer/map.rs</code> — LayeredMap<D>, push_layer(), query()</li>
|
||||
<li><code>obikindex/src/layer/evidence.rs</code> — Evidence, EvidenceWriter, encodage chunk_id:rank</li>
|
||||
<li><code>obikindex/src/layer/fingerprint.rs</code> — FingerprintVec, FingerprintVecWriter, matches()</li>
|
||||
<li><code>obikindex/src/layer/meta.rs</code> — LayerMeta, EvidenceKind (Exact / Approx { b, z })</li>
|
||||
</ul>
|
||||
<h2 id="notes">Notes</h2>
|
||||
<p>FORT RISQUE DE DÉRIVE. C'est le fichier le plus affecté par les changements récents :
|
||||
+35
-7
@@ -23,7 +23,7 @@
|
||||
|
||||
|
||||
|
||||
<title>obilayeredmap crate - obikmer</title>
|
||||
<title>obikindex layer tier - obikmer</title>
|
||||
|
||||
|
||||
|
||||
@@ -64,7 +64,7 @@
|
||||
<div data-md-component="skip">
|
||||
|
||||
|
||||
<a href="#obilayeredmap-layered-kmer-index-crate" class="md-skip">
|
||||
<a href="#obikindexlayer-the-layer-tier" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
@@ -100,7 +100,7 @@
|
||||
<div class="md-header__topic" data-md-component="header-topic">
|
||||
<span class="md-ellipsis">
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
</span>
|
||||
</div>
|
||||
@@ -749,7 +749,7 @@
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -767,7 +767,7 @@
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1352,6 +1352,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1923,9 +1951,9 @@
|
||||
|
||||
|
||||
|
||||
<h1 id="obilayeredmap-layered-kmer-index-crate">obilayeredmap — layered kmer index crate</h1>
|
||||
<h1 id="obikindexlayer-the-layer-tier">obikindex::layer — the Layer tier</h1>
|
||||
<h2 id="purpose">Purpose</h2>
|
||||
<p><code>obilayeredmap</code> implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a <code>ptr_hash</code> MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers.</p>
|
||||
<p><code>obikindex::layer</code> (the <code>layer/</code> submodule of the <code>obikindex</code> crate — a standalone <code>obilayeredmap</code> crate until 2026-08-21, folded back in alongside <code>obikpartition</code> as part of a broader <code>Index { Partition { Layer } }</code> submodule regrouping) implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a <code>ptr_hash</code> MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers.</p>
|
||||
<hr />
|
||||
<h2 id="three-usage-modes">Three usage modes</h2>
|
||||
<p>The MPHF + evidence infrastructure is the same for all modes. The <strong>payload</strong> varies.</p>
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1243,9 +1271,9 @@
|
||||
<h2 id="code-couvert">Code couvert</h2>
|
||||
<ul>
|
||||
<li><code>obikindex/src/merge.rs</code> — <code>KmerIndex::merge()</code>, validation de compatibilité d'évidence, <code>validate_evidence_compat()</code></li>
|
||||
<li><code>obikpartitionner/src/merge_layer.rs</code> — <code>merge_partition()</code>, construction de la nouvelle layer, paramètre <code>block_bits</code></li>
|
||||
<li><code>obikpartitionner/src/rebuild_layer.rs</code> — <code>rebuild_partition()</code>, paramètre <code>block_bits</code></li>
|
||||
<li><code>obilayeredmap/src/layer.rs</code> — <code>Layer::append_genome_column()</code> (PersistentCompactIntMatrix et PersistentBitMatrix)</li>
|
||||
<li><code>obikindex/src/partition/merge_layer.rs</code> — <code>merge_partition()</code>, construction de la nouvelle layer, paramètre <code>block_bits</code></li>
|
||||
<li><code>obikindex/src/partition/rebuild_layer.rs</code> — <code>rebuild_partition()</code>, paramètre <code>block_bits</code></li>
|
||||
<li><code>obikindex/src/layer/typed_layer.rs</code> — <code>Layer::append_genome_column()</code> (PersistentCompactIntMatrix et PersistentBitMatrix)</li>
|
||||
<li><code>obicompactvec/src/intmatrix.rs</code> — <code>append_column</code> pour PersistentCompactIntMatrix</li>
|
||||
<li><code>obicompactvec/src/bitmatrix.rs</code> — <code>append_column</code> pour PersistentBitMatrix</li>
|
||||
</ul>
|
||||
|
||||
@@ -733,14 +733,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1185,6 +1185,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -733,14 +733,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1180,6 +1180,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1242,8 +1270,8 @@
|
||||
<h1 id="coverage-implementationmphfmd">Coverage: implementation/mphf.md</h1>
|
||||
<h2 id="code-couvert">Code couvert</h2>
|
||||
<ul>
|
||||
<li><code>obilayeredmap/src/mphf_layer.rs</code> — type Mphf (PtrHash + CubicEps + CachelineEfVec + Xx64), construction en 2 passes, <code>build()</code>, <code>build_exact_evidence()</code>, <code>build_approx_evidence()</code>, <code>build_evidence()</code></li>
|
||||
<li><code>obikpartitionner/src/index_layer.rs</code> — <code>build_index_layer()</code> avec passage de <code>block_bits</code></li>
|
||||
<li><code>obikindex/src/layer/mphf_layer.rs</code> — type Mphf (PtrHash + CubicEps + CachelineEfVec + Xx64), construction en 2 passes, <code>build()</code>, <code>build_exact_evidence()</code>, <code>build_approx_evidence()</code>, <code>build_evidence()</code></li>
|
||||
<li><code>obikindex/src/partition/index_layer.rs</code> — <code>build_index_layer()</code> avec passage de <code>block_bits</code></li>
|
||||
</ul>
|
||||
<h2 id="notes">Notes</h2>
|
||||
<p>FORT RISQUE DE DÉRIVE. Changements récents :
|
||||
|
||||
@@ -946,14 +946,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1191,6 +1191,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1605,7 +1633,7 @@
|
||||
<h2 id="why-two-phases-are-needed">Why two phases are needed</h2>
|
||||
<p>Kmer indexing per partition proceeds in two phases. The separation is necessary because the exact number of surviving unique kmers is not known until after counting and filtering low-abundance kmers.</p>
|
||||
<h3 id="phase-1-provisional-mphf-kmer-spectrum">Phase 1 — provisional MPHF + kmer spectrum</h3>
|
||||
<p>Implemented in <code>obikpartitionner::KmerPartition::count_kmer()</code> → <code>count_partition()</code>.</p>
|
||||
<p>Implemented in <code>obikindex::partition::KmerPartition::count_kmer()</code> → <code>count_partition()</code>.</p>
|
||||
<ol>
|
||||
<li><strong>External sort</strong>: read the dereplicated superkmer file; extract the raw <code>u64</code> canonical kmer value for every kmer of every superkmer. Sort in RAM-bounded chunks (adaptive budget: 40% of available RAM ÷ n_threads, minimum 1 M kmers per chunk), then k-way merge with inline dedup. Result: <code>sorted_unique.bin</code> — a flat array of f0 distinct sorted <code>u64</code> values. Exact kmer count f0 is known at this point.</li>
|
||||
<li><strong>Build provisional MPHF</strong> (ptr_hash, same configuration as phase 2) over <code>sorted_unique.bin</code> using <code>new_from_par_iter</code>. Delete <code>sorted_unique.bin</code> immediately after. Persist to <code>mphf1.bin</code>.</li>
|
||||
@@ -1695,7 +1723,7 @@
|
||||
<hr />
|
||||
<h2 id="multilayer-index-architecture">Multilayer index architecture</h2>
|
||||
<h3 id="layer-structure">Layer structure</h3>
|
||||
<p>Each layer is a self-contained unit. See <a href="../obilayeredmap/">obilayeredmap</a> for the full on-disk layout. The MPHF-relevant files are:</p>
|
||||
<p>Each layer is a self-contained unit. See <a href="../layer_tier/">obikindex::layer</a> for the full on-disk layout. The MPHF-relevant files are:</p>
|
||||
<div class="highlight"><pre><span></span><code>layer_i/
|
||||
unitigs.bin — packed 2-bit nucleotide sequences (kmer evidence source)
|
||||
unitigs.bin.idx — random-access block index (block_bits controls granularity)
|
||||
@@ -1730,7 +1758,7 @@ MphfLayer::build_approx_evidence(dir, b, z)
|
||||
Uses open_sequential(); never writes .idx
|
||||
</code></pre></div>
|
||||
<p>There is no <code>build_evidence</code> dispatch wrapper. Callers choose the appropriate post-hoc build directly.</p>
|
||||
<p>In <code>obikpartitionner</code>, <code>build_index_layer</code> receives <code>block_bits: u8</code> from <code>IndexConfig::block_bits</code> and forwards it directly to <code>Layer::build</code> and <code>Layer::build_approx_evidence</code>.</p>
|
||||
<p>In <code>obikindex::partition</code>, <code>build_index_layer</code> receives <code>block_bits: u8</code> from <code>IndexConfig::block_bits</code> and forwards it directly to <code>Layer::build</code> and <code>Layer::build_approx_evidence</code>.</p>
|
||||
<h3 id="membership-verification">Membership verification</h3>
|
||||
<p>ptr_hash maps any input to a valid slot — it does not natively detect absent keys. Membership is verified using the evidence entry:</p>
|
||||
<ul>
|
||||
|
||||
@@ -320,11 +320,11 @@
|
||||
</a>
|
||||
</li>
|
||||
<li class="md-nav__item">
|
||||
<a class="md-nav__link" href="../obilayeredmap/">
|
||||
<a class="md-nav__link" href="../layer_tier/">
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -424,6 +424,18 @@
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="md-nav__item">
|
||||
<a class="md-nav__link" href="../partition_layer_cache/">
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
</a>
|
||||
</li>
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1247,7 +1275,7 @@
|
||||
</ul>
|
||||
<h2 id="notes">Notes</h2>
|
||||
<p>Document stable (librairie générique, peu de risque de dérive).
|
||||
Vérifier si <code>obipipeline</code> est toujours utilisé dans la phase scatter de <code>obikpartitionner</code>
|
||||
Vérifier si <code>obipipeline</code> est toujours utilisé dans la phase scatter de <code>obikindex::partition</code>
|
||||
ou s'il a été remplacé par Rayon dans certains chemins.</p>
|
||||
|
||||
|
||||
|
||||
@@ -896,14 +896,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1141,6 +1141,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -733,14 +733,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1153,6 +1153,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -733,14 +733,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1253,6 +1253,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
|
||||
|
||||
<link rel="prev" href="../obilayeredmap/">
|
||||
<link rel="prev" href="../layer_tier/">
|
||||
|
||||
|
||||
<link rel="next" href="../persistent_bit_vec/">
|
||||
@@ -733,14 +733,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1236,6 +1236,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1242,12 +1270,12 @@
|
||||
<h1 id="coverage-implementationpipelinemd">Coverage: implementation/pipeline.md</h1>
|
||||
<h2 id="code-couvert">Code couvert</h2>
|
||||
<ul>
|
||||
<li><code>obikpartitionner/src/partition.rs</code> — estimation des paramètres (phase 0)</li>
|
||||
<li><code>obikindex/src/partition/partition.rs</code> — estimation des paramètres (phase 0)</li>
|
||||
<li><code>obiskbuilder/src/iter.rs</code> — scatter : filtre entropie, extraction superkmers, routage partition (phase 1)</li>
|
||||
<li><code>obikpartitionner/src/filter.rs</code> — déduplication bucket-sort (phase 2)</li>
|
||||
<li><code>obikpartitionner/src/kmer_sort.rs</code> — tri externe + agrégation de comptages (phase 3)</li>
|
||||
<li><code>obikindex/src/partition/filter.rs</code> — déduplication bucket-sort (phase 2)</li>
|
||||
<li><code>obikindex/src/partition/kmer_sort.rs</code> — tri externe + agrégation de comptages (phase 3)</li>
|
||||
<li><code>obidebruinj/src/debruijn.rs</code> — graphe De Bruijn, extraction des unitigs (phase 5)</li>
|
||||
<li><code>obikpartitionner/src/index_layer.rs</code> — construction MPHF + évidence (phase 6), paramètre <code>block_bits</code></li>
|
||||
<li><code>obikindex/src/partition/index_layer.rs</code> — construction MPHF + évidence (phase 6), paramètre <code>block_bits</code></li>
|
||||
<li><code>obikindex/src/index.rs</code> — <code>build_layers()</code>, <code>dereplicate_and_count()</code></li>
|
||||
</ul>
|
||||
<h2 id="notes">Notes</h2>
|
||||
|
||||
@@ -857,14 +857,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1102,6 +1102,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1574,7 +1602,7 @@ branching / dead-end → unitig start or end
|
||||
counts/ ← PersistentCompactIntMatrix (with_counts only)
|
||||
</code></pre></div>
|
||||
<p><strong>Cleanup:</strong> unless <code>--keep-intermediate</code> is set, <code>remove_build_artifacts</code> deletes <code>dereplicated.skmer.zst</code>, <code>mphf1.bin</code>, and <code>counts1.bin</code> after all partitions are indexed.</p>
|
||||
<p>See <a href="../obilayeredmap/">obilayeredmap</a> and <a href="../mphf/">MPHF selection</a> for data structure details.</p>
|
||||
<p>See <a href="../layer_tier/">obikindex::layer</a> and <a href="../mphf/">MPHF selection</a> for data structure details.</p>
|
||||
<p><strong>Query path (exact evidence):</strong></p>
|
||||
<div class="highlight"><pre><span></span><code>query kmer q
|
||||
→ canonical_minimizer(q) → hash → PART → part_XXXXX/
|
||||
|
||||
@@ -733,14 +733,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1224,6 +1224,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1244,7 +1272,7 @@
|
||||
<ul>
|
||||
<li><code>obikindex/src/meta.rs</code> — IndexMeta, IndexConfig (version, config, genomes)</li>
|
||||
<li><code>obikindex/src/index.rs</code> — layout sur disque : partitions/, index.meta</li>
|
||||
<li><code>obilayeredmap/src/meta.rs</code> — LayerMeta (evidence kind), PartitionMeta (n_layers)</li>
|
||||
<li><code>obikindex/src/layer/meta.rs</code> — LayerMeta (evidence kind), PartitionMeta (n_layers)</li>
|
||||
<li><code>obiskio/src/unitig_index.rs</code> — fichiers unitigs.bin + unitigs.bin.idx</li>
|
||||
</ul>
|
||||
<h2 id="notes">Notes</h2>
|
||||
|
||||
@@ -929,14 +929,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1174,6 +1174,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -835,14 +835,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1080,6 +1080,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1243,7 +1271,7 @@
|
||||
<h2 id="code-couvert">Code couvert</h2>
|
||||
<ul>
|
||||
<li><code>obiskio/src/unitig_index.rs</code> — format unitigs.bin + unitigs.bin.idx, UnitigFileWriter, UnitigFileReader, build_unitig_idx(), DEFAULT_BLOCK_BITS=0, chemin chaud block_bits=0 dans chunk_start()</li>
|
||||
<li><code>obilayeredmap/src/evidence.rs</code> — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter</li>
|
||||
<li><code>obikindex/src/layer/evidence.rs</code> — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter</li>
|
||||
<li><code>obidebruinj/src/debruijn.rs</code> — extraction unitigs, chunking à MAX_KMERS_PER_CHUNK</li>
|
||||
</ul>
|
||||
<h2 id="notes">Notes</h2>
|
||||
|
||||
@@ -957,14 +957,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../obilayeredmap/" class="md-nav__link">
|
||||
<a href="../layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1202,6 +1202,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
+30
-2
@@ -831,14 +831,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1076,6 +1076,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -917,14 +917,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1162,6 +1162,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
+30
-2
@@ -830,14 +830,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1075,6 +1075,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
Binary file not shown.
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -802,14 +802,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1047,6 +1047,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -868,14 +868,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1113,6 +1113,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -1244,14 +1244,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1489,6 +1489,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -3582,7 +3610,7 @@ diagonal entry directly, at no extra lookup cost.</p>
|
||||
<p>Mirror <code>distance.rs</code>: open the presence or count store per partition. But
|
||||
instead of a per-partition <code>partial_*</code>, run the sequential source sweep:</p>
|
||||
<div class="highlight"><pre><span></span><code>for p in 0..n_partitions: # OUTER — sequential
|
||||
open source partition p's layers (QueryLayer-style, obikpartitionner)
|
||||
open source partition p's layers (QueryLayer-style, obikindex::partition)
|
||||
enumerate distinct canonical k-mers of p (one per MPHF slot) with their
|
||||
presence/count vectors # column-major, as query stage 2
|
||||
par_iter over these source k-mers: # INNER — rayon, thread-local tally
|
||||
@@ -3605,7 +3633,7 @@ instead of a per-partition <code>partial_*</code>, run the sequential source swe
|
||||
merge thread-local tallies into global SnpTally
|
||||
</code></pre></div>
|
||||
<p>The inner lookup is precisely <code>QueryLayer::find_slot</code> +
|
||||
<code>col_value(g, slot)</code> (<code>obikpartitionner/src/query_layer.rs</code>) — reuse or factor
|
||||
<code>col_value(g, slot)</code> (<code>obikindex/src/partition/query_layer.rs</code>) — reuse or factor
|
||||
out that path rather than reimplementing MPHF access. Enumerating "all distinct
|
||||
k-mers of a partition with their vectors" is the <code>dump</code>/<code>query</code> stage-2
|
||||
column-major scan already implemented in <code>dump_layer.rs</code> /
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
@@ -1242,8 +1270,8 @@
|
||||
<h1 id="coverage-theoryindexingmd">Coverage: theory/indexing.md</h1>
|
||||
<h2 id="code-couvert">Code couvert</h2>
|
||||
<ul>
|
||||
<li><code>obikpartitionner/src/partition.rs</code> — routage par hash de minimiseur, choix des paramètres</li>
|
||||
<li><code>obikpartitionner/src/lib.rs</code> — structure KmerPartition, nombre de partitions</li>
|
||||
<li><code>obikindex/src/partition/partition.rs</code> — routage par hash de minimiseur, choix des paramètres</li>
|
||||
<li><code>obikindex/src/partition/mod.rs</code> — structure KmerPartition, nombre de partitions</li>
|
||||
</ul>
|
||||
<h2 id="notes">Notes</h2>
|
||||
<p>Vérifier que la doc mentionne bien que le nombre de partitions est une puissance de 2
|
||||
|
||||
@@ -802,14 +802,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1047,6 +1047,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -727,14 +727,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -972,6 +972,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -846,14 +846,14 @@
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||||
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
obilayeredmap crate
|
||||
obikindex layer tier
|
||||
|
||||
|
||||
|
||||
@@ -1091,6 +1091,34 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
||||
|
||||
|
||||
|
||||
<span class="md-ellipsis">
|
||||
|
||||
|
||||
Partition and layer caching (discussion)
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
|
||||
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
|
||||
## Code couvert
|
||||
|
||||
- `obilayeredmap/src/layer.rs` — Layer<D>, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix
|
||||
- `obilayeredmap/src/mphf_layer.rs` — MphfLayer, EvidenceKind (Exact / Approx), LayerEvidence enum
|
||||
- `obilayeredmap/src/map.rs` — LayeredMap<D>
|
||||
- `obilayeredmap/src/meta.rs` — LayerMeta, PartitionMeta
|
||||
- `obikindex/src/layer/typed_layer.rs` — Layer<D>, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix
|
||||
- `obikindex/src/layer/mphf_layer.rs` — MphfLayer, EvidenceKind (Exact / Approx), LayerEvidence enum
|
||||
- `obikindex/src/layer/map.rs` — LayeredMap<D>
|
||||
- `obikindex/src/layer/meta.rs` — LayerMeta, PartitionMeta
|
||||
- `obikindex/src/meta.rs` — IndexConfig (kmer_size, n_bits, with_counts, evidence, block_bits), IndexMeta
|
||||
- `obikindex/src/index.rs` — KmerIndex, build_layers
|
||||
- `obicompactvec/src/` — PersistentCompactIntMatrix, PersistentBitMatrix (DataStore implementations)
|
||||
|
||||
@@ -72,9 +72,9 @@ Each NUMA group runs its own independent adaptive pool. Workers are distributed
|
||||
|------|--------|
|
||||
| `obikindex/src/merge.rs` | Detect NUMA topology; build N `ThreadPool`s with pinned threads; assign each pre-spawned worker to a pool; wrap `merge_partition` in `pool.install()` |
|
||||
| `obikindex/src/merge.rs` | Replace `available_parallelism()` with per-NUMA core count for spawn criterion |
|
||||
| `obikpartition/src/merge_layer.rs` | No change — `merge_partition` already works inside any Rayon context |
|
||||
| `obikindex/src/partition/merge_layer.rs` | No change — `merge_partition` already works inside any Rayon context |
|
||||
| `obidebruinj/src/debruijn.rs` | No change — `par_iter` and `current_num_threads` are pool-context-aware |
|
||||
| `obikpartition/src/partition.rs` | No change — same reason |
|
||||
| `obikindex/src/partition/partition.rs` | No change — same reason |
|
||||
|
||||
## Platform guard
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ Given a set of query sequences, determine for each sequence how many of its k-me
|
||||
|
||||
## Algorithm
|
||||
|
||||
The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside `process_chunk` (`query.rs`); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (`obikpartition::query_partition_with`), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.
|
||||
The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside `process_chunk` (`query.rs`); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (`obikindex::partition::query_partition_with`), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.
|
||||
|
||||
```
|
||||
for each chunk of sequences (parallel workers via obipipeline, one call to process_chunk):
|
||||
@@ -118,11 +118,11 @@ The `-z` CLI option overrides the index metadata value. A higher z increases str
|
||||
|
||||
## Layer lookup: `MphfLayer::find`
|
||||
|
||||
`MphfLayer::open(dir, mode: &IndexMode)` receives the mode from `PartitionMeta` — no per-layer file is read. The caller (`QueryLayer`) never chooses the dispatch path: it is fixed at open time by `LayerEvidence`. See [obilayeredmap](../implementation/obilayeredmap.md) for the full `find` / `find_strict` API.
|
||||
`MphfLayer::open(dir, mode: &IndexMode)` receives the mode from `PartitionMeta` — no per-layer file is read. The caller (`QueryLayer`) never chooses the dispatch path: it is fixed at open time by `LayerEvidence`. See [obikindex::layer](../implementation/layer_tier.md) for the full `find` / `find_strict` API.
|
||||
|
||||
### `QueryLayer` variant selection
|
||||
|
||||
`QueryLayer::open` (`obikpartition/src/query_layer.rs:28-45`) only ever returns two variants — `Presence` or `Count`, checked in this order:
|
||||
`QueryLayer::open` (`obikindex/src/partition/query_layer.rs:28-45`) only ever returns two variants — `Presence` or `Count`, checked in this order:
|
||||
|
||||
| Order | Condition | Variant | Data returned per k-mer |
|
||||
|---|---|---|---|
|
||||
@@ -244,7 +244,7 @@ Fix direction: restructure `query`'s pipe with an initial `Flat` stage analogous
|
||||
`chunk_bytes = available_memory_bytes() / (n_workers * 16)` (`query.rs:407-414`) assumes a fixed ~8–16× overhead per raw input byte. But `KmerResults::new` (`query.rs:165-179`) allocates `data: Vec<u32>` sized `total_kmers_in_chunk × n_genomes` — dense, **for every k-mer position in the chunk, hit or not** — plus `win_min` and (with `--detail`) `cov`, same scaling. Real per-chunk memory is `O(n_genomes)`, not constant; the formula doesn't know `n_genomes` at all. This is the direct cause of the OOM kill on indexes with many reference genomes.
|
||||
|
||||
**4. MPHF lookup and matrix-row fetch are fused, not staged.**
|
||||
`QueryLayer::find_into` (`obikpartition/src/query_layer.rs:48-67`) does the MPHF `find` *and* the `fill_row` matrix read in one call per k-mer, inside a single-threaded loop (`query_partition_with`). There is no separation between "is this k-mer indexed" (cheap, `O(1)`, independent of `n_genomes`) and "what are its per-genome values" (the expensive, `n_genomes`-scaling part).
|
||||
`QueryLayer::find_into` (`obikindex/src/partition/query_layer.rs:48-67`) does the MPHF `find` *and* the `fill_row` matrix read in one call per k-mer, inside a single-threaded loop (`query_partition_with`). There is no separation between "is this k-mer indexed" (cheap, `O(1)`, independent of `n_genomes`) and "what are its per-genome values" (the expensive, `n_genomes`-scaling part).
|
||||
|
||||
**5. Dereplication should happen at k-mer granularity, directly — not via an intermediate superkmer-level dedup.**
|
||||
`QueryBatch::from_records` currently dereplicates at the *superkmer* level (`HashMap<RoutableSuperKmer, Vec<SKDesc>>`, `query.rs:112`). This misses redundancy between k-mers shared by *different* superkmers (read overlaps, repeats, a SNP splitting an otherwise-identical run). Superkmer *construction* (`SuperKmerIter`) stays mandatory — it is the mechanism that computes minimizers/partition routing, not an optional dedup layer — but the dedup structure built on top of it should key directly on `CanonicalKmer`, in the same pass: `HashMap<CanonicalKmer, Vec<(seq_idx, pos)>>`. This also means the MPHF `find` itself runs once per **distinct** k-mer instead of once per occurrence — a win independent of the matrix-fetch cost below.
|
||||
@@ -285,7 +285,7 @@ None of this is implemented yet — parked here as a coherent roadmap while the
|
||||
|
||||
## Implementation plan
|
||||
|
||||
Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current `query` output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning `obikmer`, `obikpartition`, and `obicompactvec` — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense `KmerResults`) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if `rapidgzip-rs` is validated (see [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen)).
|
||||
Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current `query` output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning `obikmer`, `obikindex::partition`, and `obicompactvec` — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense `KmerResults`) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if `rapidgzip-rs` is validated (see [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen)).
|
||||
|
||||
Instrumentation is deliberately sequenced *before* the I/O fix (reordering the roadmap's own listed order), because every later phase's justification rests on a measurement ("to be measured, not assumed" appears throughout the roadmap above) — without it, phases 3–5 would be undertaken on faith.
|
||||
|
||||
@@ -295,7 +295,7 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the
|
||||
|
||||
**Debug logging.** Every phase that changes an algorithmic choice (not phase 0, which *is* the logging) adds `tracing::debug!`/`trace!` at points that let a cluster run's logs answer "did this help": counts, ratios, and timings that quantify the specific claim that phase makes — e.g. phase 3 must log how many MPHF `find` calls were saved by k-mer-level dedup (the whole justification for that phase), phase 4 must log per-column scan timings, phase 5 must log actual retained-memory / sparsity ratios achieved. Prefer one structured `debug!` per chunk (fields, not prose) over free-text — the cluster logs will be the only evidence available for judging these choices, so they need to be grep/awk-able, not just readable.
|
||||
|
||||
**Unit tests.** This project's convention (`obiread`, `obikseq`, `obidebruinj`, `obicompactvec`, `obilayeredmap`, `obiskio`, `obifastwrite`) is `#[cfg(test)] #[path = "tests/<name>.rs"] mod tests;` at the bottom of the source file, with the actual test code in a sibling `src/tests/<name>.rs`. Neither `obikmer` nor `obikpartition` (the two crates phases 3 and 5 touch most) currently have a `src/tests/` directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.
|
||||
**Unit tests.** This project's convention (`obiread`, `obikseq`, `obidebruinj`, `obicompactvec`, `obikindex::layer`, `obiskio`, `obifastwrite`) is `#[cfg(test)] #[path = "tests/<name>.rs"] mod tests;` at the bottom of the source file, with the actual test code in a sibling `src/tests/<name>.rs`. Neither `obikmer` nor `obikindex::partition` (the two crates phases 3 and 5 touch most) currently have a `src/tests/` directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.
|
||||
|
||||
**Workflow (`jj`).** Work happens in a fresh `jj` commit, easy to abandon. `jj new` between phases is reasonable where it helps isolate a phase for review, but only when the working copy compiles at that point (project convention) — phase 3's internal sub-steps (batch dedup change, then `query_layer.rs` split, then the new return shape) will likely not each compile independently since they're one coupled change, so treat "commit boundary" and "plan phase boundary" as related but not forced to match 1:1; use judgement per phase rather than mechanically splitting on every bullet.
|
||||
|
||||
@@ -345,13 +345,13 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the
|
||||
- `obikmer/src/cmd/query.rs`:
|
||||
- Replace `QueryBatch::from_records`'s dedup map (`HashMap<RoutableSuperKmer, Vec<SKDesc>>`, current `query.rs:112`) with a per-partition `HashMap<CanonicalKmer, Vec<(seq_idx: u32, pos: u32)>>`, built in the same `SuperKmerIter` pass: superkmer construction and partition routing (`part_idx` from the superkmer's minimizer hash) are unchanged, only the granularity of what gets deduplicated changes — each `CanonicalKmer` within a superkmer is inserted individually instead of the whole superkmer being the dedup key.
|
||||
- **Verified**: `CanonicalKmer` (`obikseq/src/kmer.rs:390`, `pub type CanonicalKmer = CanonicalKmerOf<KLen>`) — the underlying `CanonicalKmerOf<L>` derives `Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash` (`kmer.rs:269`). Usable as a `HashMap`/`HashSet` key as-is, no change needed.
|
||||
- `obikpartition/src/query_layer.rs`:
|
||||
- `obikindex/src/partition/query_layer.rs`:
|
||||
- Split `QueryLayer::find_into` (`query_layer.rs:48-67`) into two methods: `find_slot(&self, kmer: CanonicalKmer) -> Option<usize>` (MPHF only, no matrix touch) and keep `fill_row` as-is for phase 4 to call later.
|
||||
- Replace `query_partition_with`'s inner loop (`query_layer.rs:103-113`) with a version that, for each unique `CanonicalKmer`, calls `find_slot` across the partition's layers (stopping at first hit, same as today), and instead of immediately filling a row, records `(layer_idx, slot)`.
|
||||
- New return shape for the partition-level query, replacing today's `on_hit(sk_idx, kmer_idx, row)` callback: `HashMap<layer_idx, HashMap<slot, Vec<(seq_idx, pos)>>>` (roadmap point 6) — built directly from the k-mer dedup map's `Vec<(seq_idx,pos)>` values, keyed by the resolved slot instead of the k-mer.
|
||||
- **This phase alone has no throughput benefit yet** (matrix fetch still happens, just deferred) beyond the k-mer-level dedup itself (fewer MPHF calls when queries have overlapping/repeated k-mers) — its purpose is to produce the input phase 4 needs. Land phase 3+4 together, not phase 3 alone, per the "don't split 3–5 across releases" note above.
|
||||
- Log, per chunk: total k-mer occurrences vs. unique `CanonicalKmer` count (the dedup ratio — the entire justification for this phase) and the resulting MPHF `find` call count. If the dedup ratio is close to `1.0` on real query data (little redundancy), that's the cluster run telling us this phase wasn't worth it — the logging needs to be able to say that, not just confirm the happy path.
|
||||
- **Unit tests**: create `obikmer/src/cmd/tests/query.rs` (new `src/tests/` dir for this crate, following the project's `#[cfg(test)] #[path = "tests/query.rs"] mod tests;` convention) and `obikpartition/src/tests/query_layer.rs` (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the `find_slot`/bucket-by-layer-and-slot construction against a small hand-built `QueryLayer` fixture, asserting the `(layer_idx, slot, seq_idx, pos)` tuples match what the old per-occurrence loop would have produced.
|
||||
- **Unit tests**: create `obikmer/src/cmd/tests/query.rs` (new `src/tests/` dir for this crate, following the project's `#[cfg(test)] #[path = "tests/query.rs"] mod tests;` convention) and `obikindex/src/partition/tests/query_layer.rs` (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the `find_slot`/bucket-by-layer-and-slot construction against a small hand-built `QueryLayer` fixture, asserting the `(layer_idx, slot, seq_idx, pos)` tuples match what the old per-occurrence loop would have produced.
|
||||
|
||||
### Phase 4 — Column-major matrix fetch (roadmap points 7–8) — implemented, NUMA parallelism deferred
|
||||
|
||||
@@ -359,7 +359,7 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the
|
||||
|
||||
**What shipped:**
|
||||
- `obicompactvec`: the per-column accessors this phase needed **already existed** — `PersistentCompactIntMatrix::col_view(c)` and `PersistentBitMatrix::col_view(c)` are public, and `IntSliceView::get(slot)`/`BitSliceView::get(slot)` are public — the original plan underestimated how much of this plumbing the pairwise-distance code (`dump`/`select`/`stats`) had already required. The one real gap: `PersistentBitMatrix::col_view()` panics on the `Implicit` variant (the documented mono-genome fast path, `bitmatrix.rs`). Added `PersistentBitMatrix::get(c, slot) -> u32` (`bitmatrix.rs`), a non-panicking column-major point lookup that returns `1` for `Implicit` regardless of `c` — the smallest surface needed, not a new `col_get` API from scratch.
|
||||
- `obikpartition/src/query_layer.rs`: `query_partition_with` is now two explicit stages, matching roadmap points 6–8: **stage 1** (MPHF-only, per unique k-mer, bucket hits by `(layer_idx, slot)`, emits `QueryHit::Found`) then **stage 2** (per layer with ≥1 hit, column-major: for each genome column `g` in `0..layer.n_cols().min(n_genomes)`, scan that layer's bucketed slots and call `col_value(g, slot)`, emitting `QueryHit::Value(descs, g, value)` on nonzero). `QueryHit` is a single enum delivered through one `FnMut(QueryHit)` callback — an earlier two-closure design (`on_found` + `on_value`) didn't borrow-check, since the caller's single mutable accumulator (`KmerResults`) can't be captured by two separate `FnMut` closures passed to the same call.
|
||||
- `obikindex/src/partition/query_layer.rs`: `query_partition_with` is now two explicit stages, matching roadmap points 6–8: **stage 1** (MPHF-only, per unique k-mer, bucket hits by `(layer_idx, slot)`, emits `QueryHit::Found`) then **stage 2** (per layer with ≥1 hit, column-major: for each genome column `g` in `0..layer.n_cols().min(n_genomes)`, scan that layer's bucketed slots and call `col_value(g, slot)`, emitting `QueryHit::Value(descs, g, value)` on nonzero). `QueryHit` is a single enum delivered through one `FnMut(QueryHit)` callback — an earlier two-closure design (`on_found` + `on_value`) didn't borrow-check, since the caller's single mutable accumulator (`KmerResults`) can't be captured by two separate `FnMut` closures passed to the same call.
|
||||
- `obikmer/src/cmd/query.rs`: `KmerResults::set` (row-major, whole-row-at-once) replaced by `mark_found` (stage 1: flag a position as indexed, independent of any genome's value) and `set_one` (stage 2: write one genome's value at one position). `QueryStats` extended with `n_columns_scanned`/`n_col_get_calls`, logged per chunk.
|
||||
- Total `get()`-equivalent calls are unchanged from the row-major version (`n_hits × n_cols` in the worst case, confirmed by `n_col_get_calls` in the debug log) — the win is locality (sequential access within one layer's column at a time, across `mmap`'d regions, instead of jumping across all columns per hit), exactly as predicted.
|
||||
|
||||
@@ -367,7 +367,7 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the
|
||||
Reading `obikindex/src/numa.rs`'s actual `run()` body (not just its doc comments) shows every call spawns a timer thread **plus one OS thread per worker slot on every NUMA node** (`std::thread::scope` + one `s.spawn()` per node per `max_workers`) — on the 192-core/8-NUMA reference machine, that's on the order of 190+ fresh OS threads spawned **per call**. This is fine for its actual, established usage in this codebase (`merge.rs`, `index.rs`'s `build_layers`): one `PartitionRunner::new()` + one `run()` call per command invocation, amortised over a batch of ~256 long-running partitions. It is not fine for `query`'s call pattern: `query_partition_with` runs once per `(chunk, partition)`, potentially thousands of times per second — spawning ~190 OS threads that often to scan a handful of genome columns would very likely cost far more than the row-major approach it's meant to replace. This is exactly the "resolve empirically, don't assume" composition risk the roadmap flagged, just resolved by reading the mechanism's actual cost before wiring it in, rather than by measuring a regression on the cluster after the fact.
|
||||
The column-major loop in stage 2 is therefore a **plain sequential loop** for now — it captures the whole, provable locality win (roadmap point 8's actual claim) without adding any parallelism mechanism. Genome-column-level parallelism (point 8's "bonus" axis) and partition-level parallelism (point 7) are both deferred — not abandoned. Candidates for a follow-up, once there's a concrete profiling need: (a) `rayon`'s already-warm global pool (`into_par_iter()`) for the column axis specifically — cheap to invoke repeatedly since it doesn't spawn threads per call, though it's the same "naive rayon" pattern `numa_worker_pools.md` warns about for a *different* workload (random pointer-chasing over large hash maps); a column scan's access pattern (sequential reads within one `mmap`'d region) has a different contention profile and hasn't been shown to have the same problem — needs its own measurement, not an assumption either way; (b) restructuring so `PartitionRunner` is invoked once per whole `query` run (or per large batch of chunks) rather than per `(chunk, partition)`, amortising its spawn cost the way `merge`/`build_layers` do — a bigger structural change than this phase's scope.
|
||||
- Log (implemented): `QueryStats::n_columns_scanned`/`n_col_get_calls`, folded into the existing per-chunk `debug!("k-mer dedup + column-major fetch", ...)` line (`query.rs`) alongside phase 3's dedup counters.
|
||||
- **Unit tests**: extended `obikpartition/src/tests/query_layer.rs` (phase 3's file) — `query_partition_with`'s empty/missing-index paths updated for the new `QueryStats` fields and single-callback signature.
|
||||
- **Unit tests**: extended `obikindex/src/partition/tests/query_layer.rs` (phase 3's file) — `query_partition_with`'s empty/missing-index paths updated for the new `QueryStats` fields and single-callback signature.
|
||||
- **Validation performed**: full workspace build + `cargo test --workspace`, zero failures. Functional validation against real indexes: (1) a single-genome index — output byte-identical to pre-phase-4 (same `kmer_count`/`kmer_strict_matches` on every record); (2) the existing 20-genome `benchmark/global_index_presence` index — runs correctly, `n_hits=0` for an unrelated query (expected: no shared k-mers between a plant read and a bacterial reference set), no panics, confirming the `Implicit`/multi-column bounds logic doesn't crash on a real multi-genome, mixed-format index; (3) **the critical correctness case**: built two single-sequence-pair test genomes, merged into one 2-genome index, queried with reads from both — reads from `genomeA` matched **only** `genomeA` (`kmer_count` identical to the pre-dedup occurrence count, zero leakage into `genomeB`'s column) and vice versa. This is the test that would have caught a column-index mixup, an off-by-one in `n_cols`, or cross-genome bleed from the stage-1/stage-2 split — it passed cleanly.
|
||||
- **Not yet done**: the microbenchmark comparing column-major vs. the old row-major access pattern's wall time / page-fault counters on a large-`n_genomes` layer — needs a realistically large multi-genome index and, for the page-fault counters specifically, Linux (not available from this development environment). Left for cluster validation alongside phases 1–3's own pending measurements.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
## Code couvert
|
||||
|
||||
- `obikmer/src/cmd/query.rs` — commande query, format de sortie
|
||||
- `obikpartition/src/query_layer.rs` — routage de la requête à travers les partitions
|
||||
- `obikindex/src/partition/query_layer.rs` — routage de la requête à travers les partitions
|
||||
- `obiread/src/lib.rs` — lecture des séquences d'entrée pour la requête
|
||||
|
||||
## Notes
|
||||
|
||||
@@ -94,9 +94,9 @@ Option B avoids storing kmer values and works uniformly regardless of filter sel
|
||||
|
||||
## Files to modify
|
||||
|
||||
- `src/obikpartition/src/rebuild_layer.rs` — `rebuild_partition` and `iter_src_layers`
|
||||
- `src/obikindex/src/partition/rebuild_layer.rs` — `rebuild_partition` and `iter_src_layers`
|
||||
- Possibly `src/obicompactvec/` — add column iterator API if not already present
|
||||
- `src/obilayeredmap/` — check if per-column sequential access is exposed on `SrcLayerData`
|
||||
- `src/obikindex/src/layer/` — check if per-column sequential access is exposed on `SrcLayerData`
|
||||
|
||||
## Open questions
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ one-way function, not an invertible bijection with a stored inverse. Any
|
||||
method that reconstructs a kmer from a bare slot number is wrong by
|
||||
construction, regardless of the mechanism used (MPHF re-hash, or evidence
|
||||
decode + direct unitig read). See `MphfLayer::kmer_at`
|
||||
(`obilayeredmap/src/mphf_layer.rs`) — flagged for removal, currently called
|
||||
(`obikindex/src/layer/mphf_layer.rs`) — flagged for removal, currently called
|
||||
from `obikphylo/siblings/build.rs` and `family_scan.rs` (since removed — see
|
||||
"Pending work" status below).
|
||||
|
||||
@@ -70,7 +70,7 @@ partition is unknown) and must keep going through
|
||||
## Pending work — done
|
||||
|
||||
The plan above shipped: `obikphylo` (a new crate — phylo-domain extension
|
||||
traits over `obikindex::KmerIndex`/`obilayeredmap::Layer<D>`, replacing the
|
||||
traits over `obikindex::KmerIndex`/`obikindex::layer::Layer<D>`, replacing the
|
||||
old `obikindex::siblings` module) builds and reads the annex purely in
|
||||
iteration order (`SiblingLayerExt::iter_siblings`/`iter_minorants`, both with
|
||||
batch variants, mirroring `Layer<D>`'s own `KmerIter`/`KmerBatchIter`
|
||||
@@ -652,13 +652,13 @@ BRWT-style column-correlation exploitation.
|
||||
`PersistentSparseBitMatrix` went from a validated but unused type to a
|
||||
real, selectable on-disk format:
|
||||
|
||||
- **Generic `Layer<D>`**: `obilayeredmap::Layer<D>`'s presence-only methods
|
||||
- **Generic `Layer<D>`**: `obikindex::layer::Layer<D>`'s presence-only methods
|
||||
(`n_cols`, `sub_matrix`, `fill_sub_matrix`) are generic over any
|
||||
`D: LayerData<Item = Box<[bool]>> + BinaryMatrix`, not hardcoded to
|
||||
`PersistentBitMatrix` — `PersistentSparseBitMatrix` implements
|
||||
`LayerData` (`open`/`read`) the same way. `find_slot`/`index_batch` were
|
||||
already generic over any `D: LayerData`, so they needed no change.
|
||||
Verified by `obilayeredmap`'s
|
||||
Verified by `obikindex::layer`'s
|
||||
`presence_layer_generic_over_sparse_matches_dense` test: build a dense
|
||||
presence layer, convert it to sparse via `build_from_dense`, open both
|
||||
as `Layer<PersistentBitMatrix>`/`Layer<PersistentSparseBitMatrix>` on
|
||||
@@ -668,7 +668,7 @@ real, selectable on-disk format:
|
||||
`AtomicUsize`s in test builds, not thread-local, so a test using a
|
||||
different `k` races every other test in the same crate binary; a k=11
|
||||
version of this test passed alone but failed under the full
|
||||
`obilayeredmap` suite for exactly that reason before being fixed.)
|
||||
`obikindex::layer` suite for exactly that reason before being fixed.)
|
||||
- **`obikphylo::siblings::cache::Mat`** gained a third variant,
|
||||
`SparsePresence(Layer<PersistentSparseBitMatrix>)`, alongside `Count`
|
||||
and `Presence` — every method (`find_slot`, `index_batch`,
|
||||
@@ -698,7 +698,7 @@ real, selectable on-disk format:
|
||||
dense-path test (`sibling_annex_one_sibling_each`) exactly — proves the
|
||||
sparse format round-trips through the real build pipeline
|
||||
(`PartitionCache` sparse-detection included), not just the
|
||||
`obicompactvec`/`obilayeredmap` unit layers below it.
|
||||
`obicompactvec`/`obikindex::layer` unit layers below it.
|
||||
|
||||
Full workspace `cargo test` (all crates, unit + doc tests) green after
|
||||
this change.
|
||||
@@ -746,7 +746,7 @@ Replaces the "four independent scans" problem above and implements
|
||||
(activation: either given; defaults `1.0`/`0.5` for whichever is unset).
|
||||
|
||||
Full workspace `cargo test` green after this change (167 unit tests in
|
||||
`obicompactvec`+`obilayeredmap`+`obikphylo` alone, plus every other
|
||||
`obicompactvec`+`obikindex::layer`+`obikphylo` alone, plus every other
|
||||
crate's suite, no regressions).
|
||||
|
||||
**Still open, not part of this change** (per "Correction to the 'single
|
||||
@@ -770,7 +770,7 @@ reproducible across two runs with warm disk cache), the opposite of
|
||||
query").
|
||||
|
||||
**Root cause, read from source, not measured in isolation:**
|
||||
`KmerPartition::query_partition_with` (`obikpartition/src/query_layer.rs:155-220`)
|
||||
`KmerPartition::query_partition_with` (`obikindex/src/partition/query_layer.rs:155-220`)
|
||||
is architecturally column-major: stage 2 walks `for g in 0..n_cols { for
|
||||
slot in hit_slots { layer.col_value(g, slot) } }`, documented (correctly)
|
||||
as the right locality strategy for the packed/columnar formats, where
|
||||
@@ -914,7 +914,7 @@ replaces.
|
||||
`PersistentCompactIntMatrix::nonzero_iter` added the same way (counts
|
||||
not excluded, per the earlier ask) — no native low-effort case, since no
|
||||
sparse count format exists, but on the same primitive, ready for one.
|
||||
- `KmerPartition::query_partition_with` (`obikpartition/src/query_layer.rs`):
|
||||
- `KmerPartition::query_partition_with` (`obikindex/src/partition/query_layer.rs`):
|
||||
stage 2's column-major `for g { for slot { col_value } }` replaced by one
|
||||
`layer.nonzero_iter(&slot_list)` call per layer, format-agnostic.
|
||||
- Tests: `nonzero_iter_matches_dense`, `nonzero_iter_matches_row`, and —
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
|
||||
## Code couvert
|
||||
|
||||
- `obilayeredmap/src/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches()
|
||||
- `obilayeredmap/src/mphf_layer.rs` — build_approx_evidence(dir, b, z), find_approx()
|
||||
- `obilayeredmap/src/meta.rs` — EvidenceKind::Approx { b, z }, LayerMeta
|
||||
- `obikindex/src/layer/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches()
|
||||
- `obikindex/src/layer/mphf_layer.rs` — build_approx_evidence(dir, b, z), find_approx()
|
||||
- `obikindex/src/layer/meta.rs` — EvidenceKind::Approx { b, z }, LayerMeta
|
||||
- `obikindex/src/reindex.rs` — KmerIndex::reindex(), conversion exact↔approx en place
|
||||
- `obikmer/src/cmd/reindex.rs` — CLI reindex, options --approx, -z, --evidence-bits, --fp, --block-size
|
||||
- `obikmer/src/cmd/index.rs` — resolve_approx_params(), options --approx, -z, --evidence-bits, --fp
|
||||
|
||||
@@ -303,7 +303,7 @@ This parameter has no effect on presence/absence indexes (where values are alrea
|
||||
|
||||
## Implementation
|
||||
|
||||
- **`obikpartition::filter::GroupQuorumFilter`** — implements `KmerFilter`
|
||||
- **`obikindex::partition::filter::GroupQuorumFilter`** — implements `KmerFilter`
|
||||
using pre-computed ingroup and outgroup index vectors. The heavy logic
|
||||
(predicate parsing, three-value evaluation, genome classification) happens
|
||||
once before any iteration; each k-mer row evaluation is a simple index
|
||||
@@ -314,7 +314,7 @@ This parameter has no effect on presence/absence indexes (where values are alrea
|
||||
`UnitigArgs`. `FilterArgs::build_filters()` returns a ready-to-use filter
|
||||
list.
|
||||
|
||||
- **`obikpartition::KmerPartition::iter_partition_kmers`** — accepts
|
||||
- **`obikindex::partition::KmerPartition::iter_partition_kmers`** — accepts
|
||||
`filters: &[Box<dyn KmerFilter>]` and applies them per-kmer before invoking
|
||||
the callback. `filter`, `dump`, and `unitig` all go through this single
|
||||
entry point.
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# obilayeredmap — layered kmer index crate
|
||||
# obikindex::layer — the Layer tier
|
||||
|
||||
## Purpose
|
||||
|
||||
`obilayeredmap` implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a `ptr_hash` MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers.
|
||||
`obikindex::layer` (the `layer/` submodule of the `obikindex` crate — a standalone `obilayeredmap` crate until 2026-08-21, folded back in alongside `obikpartition` as part of a broader `Index { Partition { Layer } }` submodule regrouping) implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a `ptr_hash` MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers.
|
||||
|
||||
---
|
||||
|
||||
+7
-7
@@ -1,14 +1,14 @@
|
||||
<!-- coverage sidecar — ne pas ajouter au nav mkdocs -->
|
||||
# Coverage: implementation/obilayeredmap.md
|
||||
# Coverage: implementation/layer_tier.md
|
||||
|
||||
## Code couvert
|
||||
|
||||
- `obilayeredmap/src/mphf_layer.rs` — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx()
|
||||
- `obilayeredmap/src/layer.rs` — Layer<D>, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column()
|
||||
- `obilayeredmap/src/map.rs` — LayeredMap<D>, push_layer(), query()
|
||||
- `obilayeredmap/src/evidence.rs` — Evidence, EvidenceWriter, encodage chunk_id:rank
|
||||
- `obilayeredmap/src/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, matches()
|
||||
- `obilayeredmap/src/meta.rs` — LayerMeta, EvidenceKind (Exact / Approx { b, z })
|
||||
- `obikindex/src/layer/mphf_layer.rs` — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx()
|
||||
- `obikindex/src/layer/typed_layer.rs` — Layer<D>, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column()
|
||||
- `obikindex/src/layer/map.rs` — LayeredMap<D>, push_layer(), query()
|
||||
- `obikindex/src/layer/evidence.rs` — Evidence, EvidenceWriter, encodage chunk_id:rank
|
||||
- `obikindex/src/layer/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, matches()
|
||||
- `obikindex/src/layer/meta.rs` — LayerMeta, EvidenceKind (Exact / Approx { b, z })
|
||||
|
||||
## Notes
|
||||
|
||||
@@ -4,9 +4,9 @@
|
||||
## Code couvert
|
||||
|
||||
- `obikindex/src/merge.rs` — `KmerIndex::merge()`, validation de compatibilité d'évidence, `validate_evidence_compat()`
|
||||
- `obikpartition/src/merge_layer.rs` — `merge_partition()`, construction de la nouvelle layer, paramètre `block_bits`
|
||||
- `obikpartition/src/rebuild_layer.rs` — `rebuild_partition()`, paramètre `block_bits`
|
||||
- `obilayeredmap/src/layer.rs` — `Layer::append_genome_column()` (PersistentCompactIntMatrix et PersistentBitMatrix)
|
||||
- `obikindex/src/partition/merge_layer.rs` — `merge_partition()`, construction de la nouvelle layer, paramètre `block_bits`
|
||||
- `obikindex/src/partition/rebuild_layer.rs` — `rebuild_partition()`, paramètre `block_bits`
|
||||
- `obikindex/src/layer/typed_layer.rs` — `Layer::append_genome_column()` (PersistentCompactIntMatrix et PersistentBitMatrix)
|
||||
- `obicompactvec/src/intmatrix.rs` — `append_column` pour PersistentCompactIntMatrix
|
||||
- `obicompactvec/src/bitmatrix.rs` — `append_column` pour PersistentBitMatrix
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ Kmer indexing per partition proceeds in two phases. The separation is necessary
|
||||
|
||||
### Phase 1 — provisional MPHF + kmer spectrum
|
||||
|
||||
Implemented in `obikpartition::KmerPartition::count_kmer()` → `count_partition()`.
|
||||
Implemented in `obikindex::partition::KmerPartition::count_kmer()` → `count_partition()`.
|
||||
|
||||
1. **External sort**: read the dereplicated superkmer file; extract the raw `u64` canonical kmer value for every kmer of every superkmer. Sort in RAM-bounded chunks (adaptive budget: 40% of available RAM ÷ n_threads, minimum 1 M kmers per chunk), then k-way merge with inline dedup. Result: `sorted_unique.bin` — a flat array of f0 distinct sorted `u64` values. Exact kmer count f0 is known at this point.
|
||||
2. **Build provisional MPHF** (ptr_hash, same configuration as phase 2) over `sorted_unique.bin` using `new_from_par_iter`. Delete `sorted_unique.bin` immediately after. Persist to `mphf1.bin`.
|
||||
@@ -101,7 +101,7 @@ type Mphf = PtrHash<
|
||||
|
||||
### Layer structure
|
||||
|
||||
Each layer is a self-contained unit. See [obilayeredmap](obilayeredmap.md) for the full on-disk layout. The MPHF-relevant files are:
|
||||
Each layer is a self-contained unit. See [obikindex::layer](layer_tier.md) for the full on-disk layout. The MPHF-relevant files are:
|
||||
|
||||
```
|
||||
layer_i/
|
||||
@@ -148,7 +148,7 @@ MphfLayer::build_approx_evidence(dir, b, z)
|
||||
|
||||
There is no `build_evidence` dispatch wrapper. Callers choose the appropriate post-hoc build directly.
|
||||
|
||||
In `obikpartition`, `build_index_layer` receives `block_bits: u8` from `IndexConfig::block_bits` and forwards it directly to `Layer::build` and `Layer::build_approx_evidence`.
|
||||
In `obikindex::partition`, `build_index_layer` receives `block_bits: u8` from `IndexConfig::block_bits` and forwards it directly to `Layer::build` and `Layer::build_approx_evidence`.
|
||||
|
||||
### Membership verification
|
||||
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
|
||||
## Code couvert
|
||||
|
||||
- `obilayeredmap/src/mphf_layer.rs` — type Mphf (PtrHash + CubicEps + CachelineEfVec + Xx64), construction en 2 passes, `build()`, `build_exact_evidence()`, `build_approx_evidence()`, `build_evidence()`
|
||||
- `obikpartition/src/index_layer.rs` — `build_index_layer()` avec passage de `block_bits`
|
||||
- `obikindex/src/layer/mphf_layer.rs` — type Mphf (PtrHash + CubicEps + CachelineEfVec + Xx64), construction en 2 passes, `build()`, `build_exact_evidence()`, `build_approx_evidence()`, `build_evidence()`
|
||||
- `obikindex/src/partition/index_layer.rs` — `build_index_layer()` avec passage de `block_bits`
|
||||
|
||||
## Notes
|
||||
|
||||
|
||||
@@ -9,5 +9,5 @@
|
||||
## Notes
|
||||
|
||||
Document stable (librairie générique, peu de risque de dérive).
|
||||
Vérifier si `obipipeline` est toujours utilisé dans la phase scatter de `obikpartition`
|
||||
Vérifier si `obipipeline` est toujours utilisé dans la phase scatter de `obikindex::partition`
|
||||
ou s'il a été remplacé par Rayon dans certains chemins.
|
||||
|
||||
@@ -1,5 +1,21 @@
|
||||
# Partition and layer caching (discussion)
|
||||
|
||||
**Superseded (2026-08-21):** `obikpartition` and `obilayeredmap` are no
|
||||
longer separate workspace crates — both were folded back into `obikindex`
|
||||
as submodules (`obikindex::partition`, `obikindex::layer`), alongside the
|
||||
crate's original content as `obikindex::index`, purely to reduce the
|
||||
crate count (no behavior change). Every mention of `obikpartition`/
|
||||
`obilayeredmap` as a *crate* below, and every dependency-direction
|
||||
argument phrased in terms of "which crate depends on which" (e.g. "this
|
||||
crate depends only on `obilayeredmap` and below, never on `obikindex`"),
|
||||
describes that now-superseded split-crate architecture and is kept as-is
|
||||
for historical context — read `obikpartition::X` as `obikindex::
|
||||
partition::X` and `obilayeredmap::X` as `obikindex::layer::X` throughout.
|
||||
The underlying module boundary and its rationale (Layer tier / Partition
|
||||
tier / Index tier, each depending only downward) are unchanged; only the
|
||||
crate-vs-module packaging changed. See [obikindex::layer](layer_tier.md)
|
||||
for the current module doc.
|
||||
|
||||
Status (2026-08-20, latest pass): (1) done — `obilayeredmap::Layer`
|
||||
exists, `Mat` is gone. (1b) done — `Layer::Empty`, the first non-ready
|
||||
state, added (panics on every read method). (2a) done — the
|
||||
|
||||
@@ -203,7 +203,7 @@ part_XXXXX/
|
||||
|
||||
**Cleanup:** unless `--keep-intermediate` is set, `remove_build_artifacts` deletes `dereplicated.skmer.zst`, `mphf1.bin`, and `counts1.bin` after all partitions are indexed.
|
||||
|
||||
See [obilayeredmap](obilayeredmap.md) and [MPHF selection](mphf.md) for data structure details.
|
||||
See [obikindex::layer](layer_tier.md) and [MPHF selection](mphf.md) for data structure details.
|
||||
|
||||
**Query path (exact evidence):**
|
||||
|
||||
|
||||
@@ -3,12 +3,12 @@
|
||||
|
||||
## Code couvert
|
||||
|
||||
- `obikpartition/src/partition.rs` — estimation des paramètres (phase 0)
|
||||
- `obikindex/src/partition/partition.rs` — estimation des paramètres (phase 0)
|
||||
- `obiskbuilder/src/iter.rs` — scatter : filtre entropie, extraction superkmers, routage partition (phase 1)
|
||||
- `obikpartition/src/filter.rs` — déduplication bucket-sort (phase 2)
|
||||
- `obikpartition/src/kmer_sort.rs` — tri externe + agrégation de comptages (phase 3)
|
||||
- `obikindex/src/partition/filter.rs` — déduplication bucket-sort (phase 2)
|
||||
- `obikindex/src/partition/kmer_sort.rs` — tri externe + agrégation de comptages (phase 3)
|
||||
- `obidebruinj/src/debruijn.rs` — graphe De Bruijn, extraction des unitigs (phase 5)
|
||||
- `obikpartition/src/index_layer.rs` — construction MPHF + évidence (phase 6), paramètre `block_bits`
|
||||
- `obikindex/src/partition/index_layer.rs` — construction MPHF + évidence (phase 6), paramètre `block_bits`
|
||||
- `obikindex/src/index.rs` — `build_layers()`, `dereplicate_and_count()`
|
||||
|
||||
## Notes
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
- `obikindex/src/meta.rs` — IndexMeta, IndexConfig (version, config, genomes)
|
||||
- `obikindex/src/index.rs` — layout sur disque : partitions/, index.meta
|
||||
- `obilayeredmap/src/meta.rs` — LayerMeta (evidence kind), PartitionMeta (n_layers)
|
||||
- `obikindex/src/layer/meta.rs` — LayerMeta (evidence kind), PartitionMeta (n_layers)
|
||||
- `obiskio/src/unitig_index.rs` — fichiers unitigs.bin + unitigs.bin.idx
|
||||
|
||||
## Notes
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
## Code couvert
|
||||
|
||||
- `obiskio/src/unitig_index.rs` — format unitigs.bin + unitigs.bin.idx, UnitigFileWriter, UnitigFileReader, build_unitig_idx(), DEFAULT_BLOCK_BITS=0, chemin chaud block_bits=0 dans chunk_start()
|
||||
- `obilayeredmap/src/evidence.rs` — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter
|
||||
- `obikindex/src/layer/evidence.rs` — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter
|
||||
- `obidebruinj/src/debruijn.rs` — extraction unitigs, chunking à MAX_KMERS_PER_CHUNK
|
||||
|
||||
## Notes
|
||||
|
||||
@@ -1489,7 +1489,7 @@ instead of a per-partition `partial_*`, run the sequential source sweep:
|
||||
|
||||
```text
|
||||
for p in 0..n_partitions: # OUTER — sequential
|
||||
open source partition p's layers (QueryLayer-style, obikpartition)
|
||||
open source partition p's layers (QueryLayer-style, obikindex::partition)
|
||||
enumerate distinct canonical k-mers of p (one per MPHF slot) with their
|
||||
presence/count vectors # column-major, as query stage 2
|
||||
par_iter over these source k-mers: # INNER — rayon, thread-local tally
|
||||
@@ -1513,7 +1513,7 @@ for p in 0..n_partitions: # OUTER — sequential
|
||||
```
|
||||
|
||||
The inner lookup is precisely `QueryLayer::find_slot` +
|
||||
`col_value(g, slot)` (`obikpartition/src/query_layer.rs`) — reuse or factor
|
||||
`col_value(g, slot)` (`obikindex/src/partition/query_layer.rs`) — reuse or factor
|
||||
out that path rather than reimplementing MPHF access. Enumerating "all distinct
|
||||
k-mers of a partition with their vectors" is the `dump`/`query` stage-2
|
||||
column-major scan already implemented in `dump_layer.rs` /
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
|
||||
## Code couvert
|
||||
|
||||
- `obikpartition/src/partition.rs` — routage par hash de minimiseur, choix des paramètres
|
||||
- `obikpartition/src/lib.rs` — structure KmerPartition, nombre de partitions
|
||||
- `obikindex/src/partition/partition.rs` — routage par hash de minimiseur, choix des paramètres
|
||||
- `obikindex/src/partition/mod.rs` — structure KmerPartition, nombre de partitions
|
||||
|
||||
## Notes
|
||||
|
||||
|
||||
+1
-1
@@ -47,7 +47,7 @@ nav:
|
||||
- MPHF selection: implementation/mphf.md
|
||||
- Unitig evidence encoding: implementation/unitig_evidence.md
|
||||
- Evidence elimination (discussion): implementation/evidence_elimination.md
|
||||
- obilayeredmap crate: implementation/obilayeredmap.md
|
||||
- obikindex layer tier: implementation/layer_tier.md
|
||||
- PersistentCompactIntVec: implementation/persistent_compact_int_vec.md
|
||||
- PersistentBitVec: implementation/persistent_bit_vec.md
|
||||
- Merge command: implementation/merge.md
|
||||
|
||||
Generated
+4
-84
@@ -1515,24 +1515,6 @@ dependencies = [
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "obikderep"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"niffler",
|
||||
"obikindex",
|
||||
"obikrope",
|
||||
"obikseq",
|
||||
"obilayeredmap",
|
||||
"obiskbuilder",
|
||||
"obiskio",
|
||||
"obisys",
|
||||
"rayon",
|
||||
"sysinfo",
|
||||
"tempfile",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "obikentropy"
|
||||
version = "0.1.0"
|
||||
@@ -1545,6 +1527,7 @@ name = "obikindex"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitvec",
|
||||
"cacheline-ef",
|
||||
"crossbeam-channel",
|
||||
"epserde",
|
||||
@@ -1556,11 +1539,11 @@ dependencies = [
|
||||
"obicompactvec",
|
||||
"obidebruinj",
|
||||
"obikentropy",
|
||||
"obikpartition",
|
||||
"obikrope",
|
||||
"obikseq",
|
||||
"obilayeredmap",
|
||||
"obipipeline",
|
||||
"obiread",
|
||||
"obiskbuilder",
|
||||
"obiskio",
|
||||
"obisys",
|
||||
"obitaxonomy",
|
||||
@@ -1568,6 +1551,7 @@ dependencies = [
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sysinfo",
|
||||
"tempfile",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
@@ -1583,13 +1567,10 @@ dependencies = [
|
||||
"kodama",
|
||||
"obidebruinj",
|
||||
"obifastwrite",
|
||||
"obikderep",
|
||||
"obikindex",
|
||||
"obikpartitionner",
|
||||
"obikphylo",
|
||||
"obikrope",
|
||||
"obikseq",
|
||||
"obilayeredmap",
|
||||
"obipipeline",
|
||||
"obiread",
|
||||
"obiskbuilder",
|
||||
@@ -1606,45 +1587,6 @@ dependencies = [
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "obikpartition"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"obicompactvec",
|
||||
"obikseq",
|
||||
"obilayeredmap",
|
||||
"obiskio",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "obikpartitionner"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"cacheline-ef",
|
||||
"epserde",
|
||||
"memmap2",
|
||||
"niffler",
|
||||
"obicompactvec",
|
||||
"obikderep",
|
||||
"obikindex",
|
||||
"obikrope",
|
||||
"obikseq",
|
||||
"obilayeredmap",
|
||||
"obipipeline",
|
||||
"obiread",
|
||||
"obiskbuilder",
|
||||
"obiskio",
|
||||
"obisys",
|
||||
"ptr_hash",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sysinfo",
|
||||
"tempfile",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "obikphylo"
|
||||
version = "0.1.0"
|
||||
@@ -1652,11 +1594,8 @@ dependencies = [
|
||||
"memmap2",
|
||||
"ndarray",
|
||||
"obicompactvec",
|
||||
"obikderep",
|
||||
"obikindex",
|
||||
"obikpartitionner",
|
||||
"obikseq",
|
||||
"obilayeredmap",
|
||||
"obipipeline",
|
||||
"obiread",
|
||||
"obiskbuilder",
|
||||
@@ -1688,25 +1627,6 @@ dependencies = [
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "obilayeredmap"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bitvec",
|
||||
"cacheline-ef",
|
||||
"epserde",
|
||||
"memmap2",
|
||||
"ndarray",
|
||||
"obicompactvec",
|
||||
"obikseq",
|
||||
"obiskio",
|
||||
"ptr_hash",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "obipipeline"
|
||||
version = "0.1.0"
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obikpartitionner","obiskio","obidebruinj","obilayeredmap", "obicompactvec", "obisys", "obikindex", "obitaxonomy", "obikentropy", "obikphylo", "obikpartition", "obikderep"]
|
||||
members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obiskio","obidebruinj", "obicompactvec", "obisys", "obikindex", "obitaxonomy", "obikentropy", "obikphylo"]
|
||||
[profile.release]
|
||||
debug = 1
|
||||
|
||||
@@ -129,7 +129,7 @@ pub fn pack_bit_matrix(dir: &Path) -> io::Result<()> {
|
||||
// A `matrix.pbmx` can already exist here even though columnar data is
|
||||
// still pending — e.g. copied verbatim from a merge's base source
|
||||
// before this layer was widened with more genome columns (see
|
||||
// `obikpartitionner::merge_partition`). Only skip (re-)packing if the
|
||||
// `obikindex::index::merge_layer::merge_partition`). Only skip (re-)packing if the
|
||||
// existing file already reflects the current column count; otherwise
|
||||
// the columnar files are newer and must be (re-)packed, overwriting the
|
||||
// stale one — never silently discarded as "leftover cleanup".
|
||||
|
||||
@@ -232,7 +232,7 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
||||
// A `matrix.pcmx` can already exist here even though columnar data is
|
||||
// still pending — e.g. copied verbatim from a merge's base source
|
||||
// before this layer was widened with more genome columns (see
|
||||
// `obikpartitionner::merge_partition`). Only skip (re-)packing if the
|
||||
// `obikindex::index::merge_layer::merge_partition`). Only skip (re-)packing if the
|
||||
// existing file already reflects the current column count; otherwise
|
||||
// the columnar files are newer and must be (re-)packed, overwriting the
|
||||
// stale one — never silently discarded as "leftover cleanup".
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::{fs, io, path::Path};
|
||||
|
||||
/// Lightweight metadata stored at the layer level (`layer_meta.json`).
|
||||
///
|
||||
/// Written by `obilayeredmap::MphfLayer::build` alongside `mphf.bin`.
|
||||
/// Written by `obikindex::layer::MphfLayer::build` alongside `mphf.bin`.
|
||||
/// Read by `PersistentBitMatrix::open` to determine `n_rows` for the
|
||||
/// implicit (mono-genome presence/absence) case.
|
||||
pub struct LayerMeta {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! How a persistent matrix's data is physically laid out on disk —
|
||||
//! orthogonal to *what* it stores (count vs. presence, `obilayeredmap`'s
|
||||
//! orthogonal to *what* it stores (count vs. presence, `obikindex::layer`'s
|
||||
//! `LayerContent` concern, one crate up). `PersistentCompactIntMatrix` only
|
||||
//! ever reports `Columnar`/`Packed`; `PersistentBitMatrix` is the only type
|
||||
//! that can also report `Sparse`/`Implicit`.
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
[package]
|
||||
name = "obikderep"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
niffler = "3.0.0"
|
||||
obikseq = { path = "../obikseq" }
|
||||
obikindex = { path = "../obikindex" }
|
||||
obilayeredmap = { path = "../obilayeredmap" }
|
||||
obiskio = { path = "../obiskio" }
|
||||
obisys = { path = "../obisys" }
|
||||
rayon = "1"
|
||||
sysinfo = "0.39"
|
||||
tracing = "0.1.44"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
obikseq = { path = "../obikseq", features = ["test-utils"] }
|
||||
obikrope = { path = "../obikrope" }
|
||||
obiskbuilder = { path = "../obiskbuilder" }
|
||||
@@ -5,15 +5,15 @@ edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
obikseq = { path = "../obikseq" }
|
||||
obikpartition = { path = "../obikpartition" }
|
||||
obitaxonomy = { path = "../obitaxonomy" }
|
||||
obiskio = { path = "../obiskio" }
|
||||
obisys = { path = "../obisys" }
|
||||
obicompactvec = { path = "../obicompactvec" }
|
||||
obilayeredmap = { path = "../obilayeredmap" }
|
||||
obidebruinj = { path = "../obidebruinj" }
|
||||
obipipeline = { path = "../obipipeline" }
|
||||
obikentropy = { path = "../obikentropy" }
|
||||
obiread = { path = "../obiread" }
|
||||
obiskbuilder = { path = "../obiskbuilder" }
|
||||
cacheline-ef = "1.1"
|
||||
epserde = "0.8"
|
||||
ptr_hash = "1.1"
|
||||
@@ -26,10 +26,13 @@ serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
indicatif = "0.18"
|
||||
tracing = "0.1.44"
|
||||
bitvec = "1"
|
||||
sysinfo = "0.39"
|
||||
hwlocality = { version = "1.0.0-alpha.11", features = ["vendored"], optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
obiread = { path = "../obiread" }
|
||||
obikseq = { path = "../obikseq", features = ["test-utils"] }
|
||||
obikrope = { path = "../obikrope" }
|
||||
tempfile = "3"
|
||||
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
|
||||
anyhow = "1"
|
||||
|
||||
+7
-6
@@ -1,6 +1,7 @@
|
||||
//! Per-partition dereplication mechanics — private to this crate.
|
||||
//! [`crate::Dereplicator`] is the public entry point; this module is the
|
||||
//! two-phase split+merge algorithm it runs once per partition.
|
||||
//! Per-partition dereplication mechanics — private to this module.
|
||||
//! [`crate::algorithms::dereplicator::Dereplicator`] is the public entry
|
||||
//! point; this module is the two-phase split+merge algorithm it runs once
|
||||
//! per partition.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
@@ -13,13 +14,13 @@ use niffler::send::compression::Format;
|
||||
use niffler::Level;
|
||||
use obikseq::superkmer::SuperKmer;
|
||||
use obikseq::Sequence;
|
||||
use obilayeredmap::{dereplicated_superkmers_path, raw_superkmers_path};
|
||||
use crate::layer::{dereplicated_superkmers_path, raw_superkmers_path};
|
||||
use obiskio::{SKFileMeta, SKFileReader, SKFileWriter, SKResult};
|
||||
|
||||
/// Scratch-file extension for this algorithm's own intermediate split
|
||||
/// buckets — never read by anything outside [`dereplicate_partition`],
|
||||
/// unlike `raw`/`dereplicated` (see `obilayeredmap::{raw_superkmers_path,
|
||||
/// dereplicated_superkmers_path}`, the actual cross-crate contract).
|
||||
/// unlike `raw`/`dereplicated` (see `crate::layer::{raw_superkmers_path,
|
||||
/// dereplicated_superkmers_path}`, the actual cross-module contract).
|
||||
const TEMP_EXT: &str = "skmer.zst";
|
||||
|
||||
/// Estimate the number of in-memory buckets needed to deduplicate the
|
||||
@@ -1,26 +1,22 @@
|
||||
//! Superkmer dereplication — the second stage of the indexing pipeline,
|
||||
//! after `obikpartitionner::PartitionRouter::run` (scatter) has written
|
||||
//! each partition's raw superkmer file, before
|
||||
//! `obikpartitionner::PartitionRouter::count_kmer` (counting) reads the
|
||||
//! result. One algorithm, one crate — see
|
||||
//! `DevDocMD/implementation/partition_layer_cache.md`'s "on avance pas à
|
||||
//! pas" note: `obikpartitionner` used to also own dereplication and
|
||||
//! counting; this crate is step one of splitting that bundle apart,
|
||||
//! deliberately one algorithm at a time rather than all at once, so a
|
||||
//! shared `Algorithm` pattern can be factored out later from real
|
||||
//! examples instead of guessed at up front.
|
||||
//! Superkmer dereplication — the sibling algorithm to
|
||||
//! [`crate::algorithms::partitionner`]: runs after
|
||||
//! `partitionner::PartitionRouter::run` (scatter) has written each
|
||||
//! partition's raw superkmer file, before
|
||||
//! `partitionner::PartitionRouter::count_kmer` (counting) reads the
|
||||
//! result — see `DevDocMD/implementation/partition_layer_cache.md`.
|
||||
|
||||
mod dereplicate;
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use niffler::Level;
|
||||
use obikindex::KmerIndex;
|
||||
use obiskio::SKResult;
|
||||
use obisys::Progress;
|
||||
use rayon::prelude::*;
|
||||
use sysinfo::System;
|
||||
|
||||
use crate::index::KmerIndex;
|
||||
|
||||
use dereplicate::{dereplicate_partition, optimal_buckets};
|
||||
|
||||
/// Deduplicates every partition's raw superkmer file in place, replacing
|
||||
@@ -68,8 +64,8 @@ impl<'a> Dereplicator<'a> {
|
||||
/// `obisys::TracedBar`'s own methods take `&self`, not `&mut self`).
|
||||
/// `total: Some(n_partitions)` — known up front here, unlike
|
||||
/// `PartitionRouter::run`'s bases-processed count, so the caller can
|
||||
/// render an actual progress bar rather than a spinner. This crate
|
||||
/// never renders anything itself — see the module docs.
|
||||
/// render an actual progress bar rather than a spinner. This algorithm
|
||||
/// never renders anything itself.
|
||||
pub fn run(&self, on_progress: Option<impl Fn(Progress) + Sync>) -> SKResult<()> {
|
||||
let level = self.level;
|
||||
let sys = System::new_all();
|
||||
@@ -88,7 +84,7 @@ impl<'a> Dereplicator<'a> {
|
||||
.map(|i| {
|
||||
let dir = self.index.layer_dir(i, 0);
|
||||
let result = if dir.exists() {
|
||||
let raw_path = obilayeredmap::raw_superkmers_path(&dir);
|
||||
let raw_path = crate::layer::raw_superkmers_path(&dir);
|
||||
let n_buckets = optimal_buckets(&raw_path, available_per_thread);
|
||||
dereplicate_partition(&dir, level, n_buckets)
|
||||
} else {
|
||||
@@ -0,0 +1,9 @@
|
||||
//! Indexing-pipeline algorithms: code that operates on a
|
||||
//! [`crate::index::KmerIndex`] to build or transform its content, as
|
||||
//! opposed to the `index`/`partition`/`layer` modules, which are the data
|
||||
//! model itself. Each algorithm is its own submodule: [`partitionner`]
|
||||
//! (routing raw super-kmers into partitions, then counting), [`dereplicator`]
|
||||
//! (deduplicating a partition's raw super-kmers before counting).
|
||||
|
||||
pub mod dereplicator;
|
||||
pub mod partitionner;
|
||||
+1
-1
@@ -11,7 +11,7 @@ use obiskio::{SKFileReader, SKResult};
|
||||
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64};
|
||||
use tracing::debug;
|
||||
|
||||
use crate::kmer_sort::sort_unique_kmers;
|
||||
use super::kmer_sort::sort_unique_kmers;
|
||||
|
||||
pub(super) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
//! K-mer partitioning: routing super-kmers into per-partition, layer-0
|
||||
//! files, and counting unique canonical k-mers. Dereplication of the raw
|
||||
//! super-kmers in between is [`crate::algorithms::dereplicator`], a
|
||||
//! sibling algorithm, not a step of this one — see
|
||||
//! `DevDocMD/implementation/partition_layer_cache.md`.
|
||||
//!
|
||||
//! Submodules: [`router`] (`PartitionRouter`, `KmerSpectrum`, the
|
||||
//! routing/counting lifecycle API — partition/layer path naming itself
|
||||
//! lives on `crate::index::KmerIndex`, not here), [`count`] (unique-kmer
|
||||
//! enumeration, MPHF, abundance counting), `kmer_sort` (external sort
|
||||
//! support for `count`).
|
||||
|
||||
mod count;
|
||||
mod kmer_sort;
|
||||
mod router;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use router::{KmerSpectrum, PartitionRouter};
|
||||
+12
-12
@@ -6,9 +6,9 @@ use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use obikindex::KmerIndex;
|
||||
use crate::index::KmerIndex;
|
||||
use obikseq::RoutableSuperKmer;
|
||||
use obilayeredmap::Layer;
|
||||
use crate::layer::Layer;
|
||||
use obiskio::SKResult;
|
||||
use obisys::{progress_bar, Progress};
|
||||
use rayon::prelude::*;
|
||||
@@ -22,7 +22,7 @@ use obiskio::SKFileWriter;
|
||||
use obipipeline::{throttle, ThrottleGuard, Throttled};
|
||||
use obiread::NucPage;
|
||||
|
||||
use crate::kmer_sort::chunk_size_from_ram;
|
||||
use super::kmer_sort::chunk_size_from_ram;
|
||||
|
||||
use super::count::count_partition;
|
||||
|
||||
@@ -73,14 +73,14 @@ impl Drop for GuardedIter {
|
||||
// ── PartitionRouter ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Routes raw super-kmers into per-partition, layer-0 files, then
|
||||
/// dereplicates and counts them — this crate's entire job now that
|
||||
/// layer/query/merge/select/rebuild/dump/distance concerns have moved to
|
||||
/// `obikindex` (they operate on built layers, which don't exist yet at this
|
||||
/// stage — see `DevDocMD/implementation/partition_layer_cache.md`).
|
||||
/// dereplicates (via the sibling [`crate::algorithms::dereplicator`]
|
||||
/// algorithm) and counts them — the entry point of the indexing pipeline,
|
||||
/// operating on layer-0 content that doesn't exist yet at this stage (see
|
||||
/// `DevDocMD/implementation/partition_layer_cache.md`).
|
||||
///
|
||||
/// Holds `&mut KmerIndex` — this is an algorithm operating on an index, not
|
||||
/// a data structure of its own; it owns no path-naming knowledge (every
|
||||
/// path comes from `index.index_dir`/`obilayeredmap::layer_dir`/
|
||||
/// path comes from `index.index_dir`/`crate::layer::layer_dir`/
|
||||
/// `Layer::create`), only the transient routing/dereplication/counting
|
||||
/// state a run needs.
|
||||
///
|
||||
@@ -291,7 +291,7 @@ impl<'a> PartitionRouter<'a> {
|
||||
.into_par_iter()
|
||||
.map(|i| {
|
||||
let dir = self.layer0_dir(i);
|
||||
let dedup_path = obilayeredmap::dereplicated_superkmers_path(&dir);
|
||||
let dedup_path = crate::layer::dereplicated_superkmers_path(&dir);
|
||||
if !dedup_path.exists() {
|
||||
pb.inc(1);
|
||||
return Ok(());
|
||||
@@ -343,9 +343,9 @@ impl<'a> PartitionRouter<'a> {
|
||||
/// Directory of partition `i`'s layer 0 — every raw/dereplicated
|
||||
/// superkmer file and provisional `mphf1.bin`/`counts1.bin` this router
|
||||
/// produces lives here, alongside where `build_index_layer`
|
||||
/// (`obikindex`) will later turn it into the real layer 0.
|
||||
/// (`crate::index`) will later turn it into the real layer 0.
|
||||
fn layer0_dir(&self, i: usize) -> PathBuf {
|
||||
obilayeredmap::layer_dir(&self.index.index_dir(i), 0)
|
||||
crate::layer::layer_dir(&self.index.index_dir(i), 0)
|
||||
}
|
||||
|
||||
fn check_not_closed(&self) -> SKResult<()> {
|
||||
@@ -360,7 +360,7 @@ impl<'a> PartitionRouter<'a> {
|
||||
if self.writers[partition].is_none() {
|
||||
let dir = self.layer0_dir(partition);
|
||||
Layer::create(&dir).map_err(|e| io::Error::other(e.to_string()))?;
|
||||
let file_path = obilayeredmap::raw_superkmers_path(&dir);
|
||||
let file_path = crate::layer::raw_superkmers_path(&dir);
|
||||
let writer = SKFileWriter::create_with(file_path, Format::Zstd, self.level)?;
|
||||
self.writers[partition] = Some(writer);
|
||||
}
|
||||
+4
-4
@@ -1,11 +1,11 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
|
||||
use obikderep::Dereplicator;
|
||||
use obikindex::{IndexConfig, KmerIndex};
|
||||
use crate::algorithms::dereplicator::Dereplicator;
|
||||
use crate::{IndexConfig, KmerIndex};
|
||||
use crate::layer::IndexMode;
|
||||
use obikrope::Rope;
|
||||
use obikseq::SuperKmer;
|
||||
use obilayeredmap::IndexMode;
|
||||
use obiskbuilder::build_superkmers;
|
||||
|
||||
use super::count::count_partition;
|
||||
@@ -69,7 +69,7 @@ fn pipeline_counts(seqs: &[&[u8]]) -> (u64, u64) {
|
||||
Dereplicator::new(&index).run(None::<fn(obisys::Progress)>).unwrap();
|
||||
|
||||
let part_dir = index.layer_dir(0, 0);
|
||||
let dedup_path = obilayeredmap::dereplicated_superkmers_path(&part_dir);
|
||||
let dedup_path = crate::layer::dereplicated_superkmers_path(&part_dir);
|
||||
if !dedup_path.exists() {
|
||||
return (0, 0);
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
use std::path::Path;
|
||||
|
||||
use obicompactvec::{PersistentBitVecBuilder, PersistentCompactIntVecBuilder};
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use obilayeredmap::{layer_dir, IndexMode, OLMError};
|
||||
use crate::layer::meta::PartitionMeta;
|
||||
use crate::layer::{layer_dir, IndexMode, OLMError};
|
||||
use obiskio::{SKError, SKResult};
|
||||
|
||||
// ── olm_to_sk ────────────────────────────────────────────────────────────────
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user