Bump obikmer version to 1.1.38 and add memory footprint logging

Updates Cargo.toml version from 1.1.37 to 1.1.38. Adds explicit memory footprint estimation for the `by_partition` k-mer dedup HashMap by computing capacity-based byte sizes for map slots and stored descriptors. These metrics are logged via `debug!` to track actual memory pressure during chunk processing.

.gitea/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  pull_request:
+    branches: ['main']
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: src
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            src/target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('src/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+
+      - name: Build
+        run: cargo build --release
+
+      - name: Test
+        run: cargo test --release

.gitea/workflows/release.yml

@@ -0,0 +1,127 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  create-release:
+    runs-on: ubuntu-latest
+    outputs:
+      release_id: ${{ steps.create.outputs.release_id }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Create Gitea release
+        id: create
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEATOKEN }}
+          TAG: ${{ github.ref_name }}
+        run: |
+          sudo apt-get update -qq && sudo apt-get install -y -qq jq
+          body=$(git for-each-ref --format='%(contents)' "refs/tags/$TAG")
+          release_id=$(curl -s -X POST \
+            "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases" \
+            -H "Authorization: token $GITEA_TOKEN" \
+            -H "Content-Type: application/json" \
+            -d "{\"tag_name\":\"$TAG\",\"name\":\"$TAG\",\"body\":$(echo "$body" | jq -Rs .)}" | jq -r '.id')
+          echo "release_id=$release_id" >> $GITHUB_OUTPUT
+
+  build-linux-x86_64:
+    needs: create-release
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: src
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust + zigbuild
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+          sudo apt-get update -qq && sudo apt-get install -y -qq jq
+          pip install ziglang --quiet --break-system-packages
+          $HOME/.cargo/bin/cargo install cargo-zigbuild
+          $HOME/.cargo/bin/rustup target add x86_64-unknown-linux-musl
+
+      - name: Create musl C/C++ wrappers
+        run: |
+          ZIG=$(python3 -c "import ziglang, os; print(os.path.join(os.path.dirname(ziglang.__file__), 'zig'))")
+          printf '#!/bin/sh\nexec "%s" cc -target x86_64-linux-musl "$@"\n' "$ZIG" | sudo tee /usr/local/bin/x86_64-linux-musl-gcc > /dev/null
+          printf '#!/bin/sh\nexec "%s" c++ -target x86_64-linux-musl "$@"\n' "$ZIG" | sudo tee /usr/local/bin/x86_64-linux-musl-g++ > /dev/null
+          sudo chmod +x /usr/local/bin/x86_64-linux-musl-gcc /usr/local/bin/x86_64-linux-musl-g++
+
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            src/target
+          key: linux-musl-cargo-${{ hashFiles('src/Cargo.lock') }}
+          restore-keys: linux-musl-cargo-
+
+      - name: Build static binary
+        env:
+          PKG_CONFIG_ALLOW_CROSS: "1"
+        run: cargo zigbuild --release --target x86_64-unknown-linux-musl
+
+      - name: Prepare and upload artifact
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEATOKEN }}
+          RELEASE_ID: ${{ needs.create-release.outputs.release_id }}
+        run: |
+          mkdir -p /tmp/dist
+          cp target/x86_64-unknown-linux-musl/release/obikmer /tmp/dist/obikmer-linux-x86_64
+          strip /tmp/dist/obikmer-linux-x86_64
+          curl -s -X POST \
+            "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases/$RELEASE_ID/assets" \
+            -H "Authorization: token $GITEA_TOKEN" \
+            -F "attachment=@/tmp/dist/obikmer-linux-x86_64"
+
+  build-macos-arm64:
+    needs: create-release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Login to registry
+        run: echo "${{ secrets.REGISTRYTOKEN }}" | docker login registry.metabarcoding.org -u ${{ secrets.REGISTRYUSER }} --password-stdin
+
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            src/target
+          key: macos-arm64-cargo-${{ hashFiles('src/Cargo.lock') }}
+          restore-keys: macos-arm64-cargo-
+
+      - name: Build macOS binary
+        run: |
+          CID=$(docker create \
+            -w /src/src \
+            registry.metabarcoding.org/cibuilder/rustcrossosx:latest \
+            cargo build --release --target aarch64-apple-darwin --no-default-features)
+          docker cp . "$CID:/src"
+          docker start -a "$CID"
+          STATUS=$(docker wait "$CID")
+          mkdir -p /tmp/dist
+          docker cp "$CID:/src/src/target/aarch64-apple-darwin/release/obikmer" /tmp/dist/obikmer-macos-arm64
+          docker rm "$CID" > /dev/null
+          [ "$STATUS" -eq 0 ]
+
+      - name: Prepare and upload artifact
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEATOKEN }}
+          RELEASE_ID: ${{ needs.create-release.outputs.release_id }}
+        run: |
+          curl -s -X POST \
+            "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases/$RELEASE_ID/assets" \
+            -H "Authorization: token $GITEA_TOKEN" \
+            -F "attachment=@/tmp/dist/obikmer-macos-arm64"

.gitignore

@@ -8,4 +8,18 @@ data-stress
 *.pb
 ./**/*.json
 *.bin
+*.log
 Betula_exilis--IGA-24-33
+benchmark/genomes
+benchmark/simulated_data
+benchmark/specimen_index_presence
+benchmark/specimen_index_count
+benchmark/global_index_presence
+benchmark/all_specific
+benchmark/global_index_count
+benchmark/stats
+benchmark/reference_index
+benchmark/reference_dist
+benchmark/obikmer_dist
+benchmark/specific_index_count
+benchmark/specific_index_presence

.serena/.gitignore

@@ -0,0 +1,2 @@
+/cache
+/project.local.yml

.serena/project.yml

@@ -0,0 +1,133 @@
+# the name by which the project can be referenced within Serena
+project_name: "obikmer"
+
+
+# list of languages for which language servers are started; choose from:
+#   al                  angular             ansible             bash                clojure
+#   cpp                 cpp_ccls            crystal             csharp              csharp_omnisharp
+#   dart                elixir              elm                 erlang              fortran
+#   fsharp              go                  groovy              haskell             haxe
+#   hlsl                html                java                json                julia
+#   kotlin              lean4               lua                 luau                markdown
+#   matlab              msl                 nix                 ocaml               pascal
+#   perl                php                 php_phpactor        powershell          python
+#   python_jedi         python_ty           r                   rego                ruby
+#   ruby_solargraph     rust                scala               scss                solidity
+#   svelte              swift               systemverilog       terraform           toml
+#   typescript          typescript_vts      vue                 yaml                zig
+#   (This list may be outdated. For the current list, see values of Language enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py
+#   For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
+#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
+#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
+#   - For Free Pascal/Lazarus, use pascal
+# Special requirements:
+#   Some languages require additional setup/installations.
+#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
+# When using multiple languages, the first language server that supports a given file will be used for that file.
+# The first language is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+languages:
+- rust
+
+# the encoding used by text files in the project
+# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
+encoding: "utf-8"
+
+# line ending convention to use when writing source files.
+# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
+# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
+line_ending:
+
+# The language backend to use for this project.
+# If not set, the global setting from serena_config.yml is used.
+# Valid values: LSP, JetBrains
+# Note: the backend is fixed at startup. If a project with a different backend
+# is activated post-init, an error will be returned.
+language_backend:
+
+# whether to use project's .gitignore files to ignore files
+ignore_all_files_in_gitignore: true
+
+# advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
+# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available.
+# No documentation on options means no options are available.
+ls_specific_settings: {}
+
+# list of additional workspace folder paths for cross-package reference support (e.g. in monorepos).
+# Paths can be absolute or relative to the project root.
+# Each folder is registered as an LSP workspace folder, enabling language servers to discover
+# symbols and references across package boundaries.
+# Currently supported for: TypeScript.
+# Example:
+#   additional_workspace_folders:
+#     - ../sibling-package
+#     - ../shared-lib
+additional_workspace_folders: []
+
+# list of additional paths to ignore in this project.
+# Same syntax as gitignore, so you can use * and **.
+# Note: global ignored_paths from serena_config.yml are also applied additively.
+ignored_paths: []
+
+# whether the project is in read-only mode
+# If set to true, all editing tools will be disabled and attempts to use them will result in an error
+# Added on 2025-04-18
+read_only: false
+
+# list of tool names to exclude.
+# This extends the existing exclusions (e.g. from the global configuration)
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+excluded_tools: []
+
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+fixed_tools: []
+
+# list of mode names that are to be activated by default, overriding the setting in the global configuration.
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
+# for this project.
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+default_modes:
+
+# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+added_modes:
+
+# initial prompt for the project. It will always be given to the LLM upon activating the project
+# (contrary to the memories, which are loaded on demand).
+initial_prompt: ""
+
+# time budget (seconds) per tool call for the retrieval of additional symbol information
+# such as docstrings or parameter information.
+# This overrides the corresponding setting in the global configuration; see the documentation there.
+# If null or missing, use the setting from the global configuration.
+symbol_info_budget:
+
+# list of regex patterns which, when matched, mark a memory entry as read‑only.
+# Extends the list from the global configuration, merging the two lists.
+read_only_memory_patterns: []
+
+# list of regex patterns for memories to completely ignore.
+# Matching memories will not appear in list_memories or activate_project output
+# and cannot be accessed via read_memory or write_memory.
+# To access ignored memory files, use the read_file tool on the raw file path.
+# Extends the list from the global configuration, merging the two lists.
+# Example: ["_archive/.*", "_episodes/.*"]
+ignored_memory_patterns: []

CLAUDE.md

@@ -73,3 +73,29 @@ Lors de l'ajout de nouveaux fichiers Markdown dans `docmd/`, mettre à jour la s
 ---
 
 Je continue à poser mes questions et à guider la discussion.
+
+---
+
+## MCP Tools
+
+**Règle absolue : avant tout travail de code, appeler `mcp__serena__initial_instructions` pour charger les instructions Serena.**
+
+### Hiérarchie des outils pour ce projet Rust
+
+**Navigation et édition de code → serena en priorité**
+- Trouver un symbole, une déclaration, les implémentations d'un trait : `mcp__serena__find_symbol`, `mcp__serena__find_declaration`, `mcp__serena__find_implementations`
+- Trouver les usages d'un symbole : `mcp__serena__find_referencing_symbols`
+- Diagnostics LSP (erreurs de compilation) : `mcp__serena__get_diagnostics_for_file`
+- Vue d'ensemble d'un fichier : `mcp__serena__get_symbols_overview`
+- Modifier le corps d'une fonction/impl : `mcp__serena__replace_symbol_body`
+- Ne pas utiliser `cclsp` quand serena couvre le besoin
+
+**Analyse architecturale → jcodemunch**
+- Hotspots, couplage, dead code, dépendances entre modules
+- Utiliser avant de refactorer une zone critique
+
+**Raisonnement complexe → sequential-thinking**
+- Décisions d'architecture, choix d'algorithme, trade-offs non triviaux
+
+**Documentation de crates → context7**
+- Toujours consulter avant d'utiliser une API de bibliothèque externe

Makefile

@@ -22,6 +22,7 @@ $(MKDOCS): $(VENV)/bin/activate
 		mkdocs mkdocs-material \
 		mkdocs-mermaid2-plugin \
 		mkdocs-bibtex
+	$(PIP) install --quiet --upgrade InSilicoSeq
 
 # ── obikmer binary ───────────────────────────────────────────────────────────
 
@@ -62,3 +63,36 @@ clean-doc:
 .PHONY: clean
 clean: clean-doc
 	rm -rf $(VENV)
+
+# ── release ───────────────────────────────────────────────────────────────────
+
+CARGO_TOML := $(CARGO_DIR)/obikmer/Cargo.toml
+
+.PHONY: bump-version
+bump-version:
+	@current=$$(grep '^version = ' $(CARGO_TOML) | head -n 1 | sed 's/version = "\(.*\)"/\1/'); \
+	if [ -n "$(RELEASE)" ]; then \
+		new_version="$(RELEASE)"; \
+	else \
+		major=$$(echo $$current | cut -d. -f1); \
+		minor=$$(echo $$current | cut -d. -f2); \
+		patch=$$(echo $$current | cut -d. -f3); \
+		new_patch=$$((patch + 1)); \
+		new_version="$$major.$$minor.$$new_patch"; \
+	fi; \
+	echo "Version: $$current -> $$new_version"; \
+	sed -i.bak "s/^version = \"$$current\"/version = \"$$new_version\"/" $(CARGO_TOML) && \
+	rm $(CARGO_TOML).bak
+
+.PHONY: release
+release: bump-version
+	@jj auto-describe
+	@jj git push --change @
+	@new_version=$$(grep '^version = ' $(CARGO_TOML) | head -n 1 | sed 's/version = "\(.*\)"/\1/'); \
+	git_hash=$$(jj log -r @ --no-graph -T 'commit_id'); \
+	commits=$$(jj log -r 'latest(tags())..@' --no-graph -T 'description ++ "\n"' 2>/dev/null || \
+	           jj log --no-graph -T 'description ++ "\n"' --limit 30); \
+	notes=$$(printf 'Write concise markdown release notes for obikmer (a Rust kmer genomics tool). Be technical and direct. Base them strictly on these commit messages:\n\n%s' "$$commits" | aichat 2>/dev/null); \
+	tag_msg="$${notes:-Release v$$new_version}"; \
+	git tag -a "v$$new_version" -m "$$tag_msg" "$$git_hash" && \
+	git push origin "v$$new_version"

benchmark/Makefile

@@ -0,0 +1,230 @@
+# Requires GNU Make >= 4.3 (grouped targets &:) — use gmake on macOS
+BINARY  := ../src/target/release/obikmer
+VENV_PY := ../.venv/bin/python3
+
+GENOMES := $(wildcard genomes/*.fna.gz)
+
+# SPECIMENS, SPECIES, and the full dependency graph are generated by
+# make_deps.py from the genome FASTA headers — like .d files in C.
+# Make rebuilds deps.mk whenever genomes/ changes and restarts.
+-include deps.mk
+
+REF_NPZS              := $(SPECIMENS:%=reference_index/%.npz)
+REF_DIST_CSVS         := $(addprefix reference_dist/, \
+	shared_kmers.csv hamming_dist.csv jaccard_dist.csv \
+	bray_curtis_dist.csv relfreq_bray_curtis_dist.csv \
+	euclidean_dist.csv relfreq_euclidean_dist.csv \
+	hellinger_dist.csv hellinger_euclidean_dist.csv)
+OBIKMER_PRESENCE_DIST := $(addprefix obikmer_dist/presence/, \
+	jaccard_dist.csv jaccard_shared.csv jaccard_nj.nwk \
+	hamming_dist.csv hamming_nj.nwk)
+OBIKMER_COUNT_DIST    := $(addprefix obikmer_dist/count/, \
+	jaccard_dist.csv jaccard_shared.csv jaccard_nj.nwk \
+	bray_curtis_dist.csv bray_curtis_nj.nwk \
+	relfreq_bray_curtis_dist.csv relfreq_bray_curtis_nj.nwk \
+	euclidean_dist.csv euclidean_nj.nwk \
+	relfreq_euclidean_dist.csv relfreq_euclidean_nj.nwk \
+	hellinger_dist.csv hellinger_nj.nwk \
+	hellinger_euclidean_dist.csv hellinger_euclidean_nj.nwk)
+DIST_COMPARISON       := stats/dist_comparison/summary.csv
+PRESENCE_DONE         := $(SPECIMENS:%=specimen_index_presence/%/index.done)
+PRESENCE_STATS        := $(SPECIMENS:%=stats/indexing_presence/%.stats)
+COUNT_DONE            := $(SPECIMENS:%=specimen_index_count/%/index.done)
+COUNT_STATS           := $(SPECIMENS:%=stats/indexing_count/%.stats)
+VERIFY_PRESENCE_STATS := $(SPECIMENS:%=stats/verify_presence/%.stats)
+VERIFY_COUNT_STATS    := $(SPECIMENS:%=stats/verify_count/%.stats)
+SPECIFIC_PRESENCE_DONE  := $(SPECIES:%=specific_index_presence/%/index.done)
+SPECIFIC_PRESENCE_STATS := $(SPECIES:%=stats/specific_kmer_presence/%.stats)
+SPECIFIC_COUNT_DONE     := $(SPECIES:%=specific_index_count/%/index.done)
+SPECIFIC_COUNT_STATS    := $(SPECIES:%=stats/specific_kmer_count/%.stats)
+SIMULATED_READS := $(foreach s,$(SPECIMENS),simulated_data/$(subst --,/,$s)/reads_R1.fastq.gz)
+
+.NOTPARALLEL:
+
+.PHONY: all simulate reference reference_dist \
+        obikmer_dist obikmer_dist_presence obikmer_dist_count \
+        dist_comparison \
+        index_presence index_count \
+        aggregate_index_presence aggregate_index_count \
+        merge_presence merge_count \
+        verify_presence verify_count \
+        aggregate_verify_presence aggregate_verify_count \
+        verify_merge_presence verify_merge_count \
+        filter_presence filter_count \
+        aggregate_filter_presence aggregate_filter_count
+
+verify_merge_presence: stats/verify_merge_presence/current.csv
+verify_merge_count:    stats/verify_merge_count/current.csv
+
+all: aggregate_verify_presence aggregate_verify_count \
+     verify_merge_presence verify_merge_count \
+     aggregate_filter_presence aggregate_filter_count \
+     dist_comparison
+
+# ── dependency file ───────────────────────────────────────────────────────────
+
+deps.mk: $(GENOMES)
+	$(VENV_PY) make_deps.py $^ > $@
+
+# ── simulation ────────────────────────────────────────────────────────────────
+# Prerequisites (genome → reads) are in deps.mk; $< is the genome file.
+
+$(SIMULATED_READS):
+	bash simulate_one.sh $< $(dir $@)
+
+simulate: $(SIMULATED_READS)
+
+# ── reference kmer sets ───────────────────────────────────────────────────────
+# Prerequisites (reads → npz) are in deps.mk.
+
+reference_index/%.npz:
+	bash build_reference.sh $*
+
+reference: $(REF_NPZS)
+
+# ── reference distance matrices ───────────────────────────────────────────────
+
+$(REF_DIST_CSVS) &: $(REF_NPZS) build_reference_dist.py
+	$(VENV_PY) build_reference_dist.py
+
+reference_dist: $(REF_DIST_CSVS)
+
+# ── obikmer distance (presence index) ────────────────────────────────────────
+
+$(OBIKMER_PRESENCE_DIST) &: global_index_presence/index.done $(BINARY)
+	mkdir -p obikmer_dist/presence
+	$(BINARY) distance \
+		--output obikmer_dist/presence/jaccard \
+		--metric jaccard --shared-kmers --nj \
+		global_index_presence
+	$(BINARY) distance \
+		--output obikmer_dist/presence/hamming \
+		--metric hamming --nj \
+		global_index_presence
+
+obikmer_dist_presence: $(OBIKMER_PRESENCE_DIST)
+
+# ── obikmer distance (count index) ───────────────────────────────────────────
+
+$(OBIKMER_COUNT_DIST) &: global_index_count/index.done $(BINARY)
+	mkdir -p obikmer_dist/count
+	$(BINARY) distance \
+		--output obikmer_dist/count/jaccard \
+		--metric jaccard --shared-kmers --nj \
+		global_index_count
+	$(BINARY) distance \
+		--output obikmer_dist/count/bray_curtis \
+		--metric bray-curtis --nj \
+		global_index_count
+	$(BINARY) distance \
+		--output obikmer_dist/count/relfreq_bray_curtis \
+		--metric relfreq-bray-curtis --nj \
+		global_index_count
+	$(BINARY) distance \
+		--output obikmer_dist/count/euclidean \
+		--metric euclidean --nj \
+		global_index_count
+	$(BINARY) distance \
+		--output obikmer_dist/count/relfreq_euclidean \
+		--metric relfreq-euclidean --nj \
+		global_index_count
+	$(BINARY) distance \
+		--output obikmer_dist/count/hellinger \
+		--metric hellinger --nj \
+		global_index_count
+	$(BINARY) distance \
+		--output obikmer_dist/count/hellinger_euclidean \
+		--metric hellinger-euclidean --nj \
+		global_index_count
+
+obikmer_dist_count: $(OBIKMER_COUNT_DIST)
+
+obikmer_dist: obikmer_dist_presence obikmer_dist_count
+
+# ── distance comparison ───────────────────────────────────────────────────────
+
+$(DIST_COMPARISON): $(REF_DIST_CSVS) $(OBIKMER_PRESENCE_DIST) $(OBIKMER_COUNT_DIST) compare_all_dist.py
+	$(VENV_PY) compare_all_dist.py --out $(DIST_COMPARISON)
+
+dist_comparison: $(DIST_COMPARISON)
+
+# ── per-specimen indexing ─────────────────────────────────────────────────────
+# Prerequisites (reads → index.done + .stats) are in deps.mk.
+
+specimen_index_presence/%/index.done \
+stats/indexing_presence/%.stats &: $(BINARY)
+	bash index_one_presence.sh $*
+
+specimen_index_count/%/index.done \
+stats/indexing_count/%.stats &: $(BINARY)
+	bash index_one_count.sh $*
+
+index_presence: $(PRESENCE_DONE)
+index_count:    $(COUNT_DONE)
+
+# ── indexing stats aggregation ────────────────────────────────────────────────
+
+aggregate_index_presence: $(PRESENCE_STATS)
+	bash aggregate_stats.sh indexing_presence
+
+aggregate_index_count: $(COUNT_STATS)
+	bash aggregate_stats.sh indexing_count
+
+# ── global merge ──────────────────────────────────────────────────────────────
+
+global_index_presence/index.done: $(PRESENCE_DONE) $(BINARY)
+	bash merge_presence.sh
+
+global_index_count/index.done: $(COUNT_DONE) $(BINARY)
+	bash merge_count.sh
+
+merge_presence: global_index_presence/index.done
+merge_count:    global_index_count/index.done
+
+# ── per-specimen verification ─────────────────────────────────────────────────
+# Prerequisites (index.done + npz → .stats) are in deps.mk.
+
+stats/verify_presence/%.stats:
+	bash verify_one_presence.sh $*
+
+stats/verify_count/%.stats:
+	bash verify_one_count.sh $*
+
+verify_presence: $(VERIFY_PRESENCE_STATS)
+verify_count:    $(VERIFY_COUNT_STATS)
+
+# ── verification stats aggregation ───────────────────────────────────────────
+
+aggregate_verify_presence: $(VERIFY_PRESENCE_STATS)
+	bash aggregate_stats.sh verify_presence
+
+aggregate_verify_count: $(VERIFY_COUNT_STATS)
+	bash aggregate_stats.sh verify_count
+
+# ── species-specific indexes ──────────────────────────────────────────────────
+# Prerequisites (global index → specific index) are in deps.mk.
+
+specific_index_presence/%/index.done \
+stats/specific_kmer_presence/%.stats &: $(BINARY)
+	bash filter_one_presence.sh $*
+
+specific_index_count/%/index.done \
+stats/specific_kmer_count/%.stats &: $(BINARY)
+	bash filter_one_count.sh $*
+
+filter_presence: $(SPECIFIC_PRESENCE_DONE)
+filter_count:    $(SPECIFIC_COUNT_DONE)
+
+aggregate_filter_presence: $(SPECIFIC_PRESENCE_STATS)
+	bash aggregate_stats.sh specific_kmer_presence
+
+aggregate_filter_count: $(SPECIFIC_COUNT_STATS)
+	bash aggregate_stats.sh specific_kmer_count
+
+# ── merged index verification ─────────────────────────────────────────────────
+
+stats/verify_merge_presence/current.csv: $(REF_NPZS) global_index_presence/index.done
+	bash verify_merge_presence.sh
+
+stats/verify_merge_count/current.csv: $(REF_NPZS) global_index_count/index.done
+	bash verify_merge_count.sh

benchmark/README.md

@@ -0,0 +1,132 @@
+# Benchmark pipeline
+
+Requires **GNU Make ≥ 4.3** (grouped targets `&:`).  On macOS use `gmake`.
+
+```
+gmake all          # full pipeline
+gmake simulate     # simulation only
+gmake reference    # reference kmer sets only
+```
+
+## Pipeline overview
+
+```mermaid
+flowchart TD
+    GENOMES["genomes/*.fna.gz"]
+    BIN["obikmer binary"]
+
+    GENOMES --> simulate
+    simulate --> simdata[("simulated_data/")]
+
+    simdata --> reference
+    reference --> refnpz[("reference_index/*.npz")]
+
+    subgraph presence ["Presence track"]
+        simdata  --> index_presence
+        BIN      --> index_presence
+        index_presence --> pres_done[("specimen_index_presence/")]
+        index_presence --> pres_istats[("stats/indexing_presence/")]
+        pres_istats --> aggregate_index_presence
+
+        pres_done --> merge_presence
+        BIN       --> merge_presence
+        merge_presence --> gpres[("global_index_presence/")]
+
+        refnpz    --> verify_presence
+        pres_done --> verify_presence
+        verify_presence --> vpres_stats[("stats/verify_presence/")]
+        vpres_stats --> aggregate_verify_presence
+
+        gpres --> filter_presence
+        BIN   --> filter_presence
+        filter_presence --> spec_pres[("specific_index_presence/")]
+        filter_presence --> spec_pres_stats[("stats/specific_kmer_presence/")]
+        spec_pres_stats --> aggregate_filter_presence
+
+        refnpz --> verify_merge_presence
+        gpres  --> verify_merge_presence
+        verify_merge_presence --> vmp[("stats/verify_merge_presence/")]
+    end
+
+    subgraph count ["Count track"]
+        simdata --> index_count
+        BIN     --> index_count
+        index_count --> count_done[("specimen_index_count/")]
+        index_count --> count_istats[("stats/indexing_count/")]
+        count_istats --> aggregate_index_count
+
+        count_done --> merge_count
+        BIN        --> merge_count
+        merge_count --> gcount[("global_index_count/")]
+
+        refnpz     --> verify_count
+        count_done --> verify_count
+        verify_count --> vcount_stats[("stats/verify_count/")]
+        vcount_stats --> aggregate_verify_count
+
+        gcount --> filter_count
+        BIN    --> filter_count
+        filter_count --> spec_count[("specific_index_count/")]
+        filter_count --> spec_count_stats[("stats/specific_kmer_count/")]
+        spec_count_stats --> aggregate_filter_count
+
+        refnpz --> verify_merge_count
+        gcount --> verify_merge_count
+        verify_merge_count --> vmc[("stats/verify_merge_count/")]
+    end
+
+    aggregate_verify_presence  --> all
+    aggregate_verify_count     --> all
+    vmp                        --> all
+    vmc                        --> all
+    all -. "$(MAKE) re-eval" .-> aggregate_filter_presence
+    all -. "$(MAKE) re-eval" .-> aggregate_filter_count
+```
+
+## Steps
+
+| Target | Script | Description |
+|---|---|---|
+| `simulate` | `simulate.sh` | Simulate sequencing reads from the reference genomes |
+| `reference` | `build_reference.sh` | Build reference kmer sets (`.npz`) from simulation truth |
+| `index_presence` | `index_one_presence.sh` | Index each specimen (presence mode) |
+| `index_count` | `index_one_count.sh` | Index each specimen (count mode) |
+| `aggregate_index_presence` | `aggregate_stats.sh` | Aggregate per-specimen indexing stats (presence) |
+| `aggregate_index_count` | `aggregate_stats.sh` | Aggregate per-specimen indexing stats (count) |
+| `merge_presence` | `merge_presence.sh` | Merge all specimen presence indexes into a global index |
+| `merge_count` | `merge_count.sh` | Merge all specimen count indexes into a global index |
+| `verify_presence` | `verify_one_presence.sh` | Verify each specimen presence index against reference |
+| `verify_count` | `verify_one_count.sh` | Verify each specimen count index against reference |
+| `aggregate_verify_presence` | `aggregate_stats.sh` | Aggregate per-specimen verification stats (presence) |
+| `aggregate_verify_count` | `aggregate_stats.sh` | Aggregate per-specimen verification stats (count) |
+| `filter_presence` | `filter_one_presence.sh` | Extract species-specific presence indexes from global index |
+| `filter_count` | `filter_one_count.sh` | Extract species-specific count indexes from global index |
+| `aggregate_filter_presence` | `aggregate_stats.sh` | Aggregate species-specific kmer stats (presence) |
+| `aggregate_filter_count` | `aggregate_stats.sh` | Aggregate species-specific kmer stats (count) |
+| `verify_merge_presence` | `verify_merge_presence.sh` | Verify global presence index against all reference sets |
+| `verify_merge_count` | `verify_merge_count.sh` | Verify global count index against all reference sets |
+
+## Directory layout
+
+```
+benchmark/
+├── genomes/                        # input reference genomes (.fna.gz)
+├── simulated_data/                 # generated by simulate
+│   └── <species>/<specimen>/
+├── reference_index/                # reference kmer sets (.npz)
+├── specimen_index_presence/        # per-specimen presence indexes
+├── specimen_index_count/           # per-specimen count indexes
+├── global_index_presence/          # merged global presence index
+├── global_index_count/             # merged global count index
+├── specific_index_presence/        # species-specific presence indexes
+├── specific_index_count/           # species-specific count indexes
+└── stats/                          # all benchmark statistics
+    ├── indexing_presence/
+    ├── indexing_count/
+    ├── verify_presence/
+    ├── verify_count/
+    ├── specific_kmer_presence/
+    ├── specific_kmer_count/
+    ├── verify_merge_presence/
+    └── verify_merge_count/
+```

benchmark/aggregate_stats.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Usage: aggregate_stats.sh TYPE
+# TYPE = indexing_presence | indexing_count | verify_presence | verify_count
+#
+# Reads all stats/TYPE/*.stats files (one CSV data row each, no header).
+# Creates a new stats/TYPE/run_NNN.csv only if any .stats file is newer than
+# the most recent run CSV (idempotent when nothing changed).
+set -euo pipefail
+
+TYPE="$1"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STATS_DIR="${SCRIPT_DIR}/stats/${TYPE}"
+
+case "${TYPE}" in
+    indexing_presence|indexing_count)
+        HEADER="run,species,strain,scatter_wall_s,scatter_rss_b,dereplicate_wall_s,dereplicate_rss_b,count_kmer_wall_s,count_kmer_rss_b,index_wall_s,index_rss_b,total_wall_s,total_rss_b"
+        ;;
+    verify_presence)
+        HEADER="run,species,strain,ref_kmers,idx_kmers,false_neg,false_pos,fn_pct,fp_pct"
+        ;;
+    verify_count)
+        HEADER="run,species,strain,ref_kmers,idx_kmers,false_neg,false_pos,count_mismatch,fn_pct,fp_pct,cm_pct"
+        ;;
+    specific_kmer_presence|specific_kmer_count)
+        HEADER="run,species,rebuild_wall_s,rebuild_rss_b,pack_wall_s,pack_rss_b,filter_total_wall_s,filter_total_rss_b,select_wall_s,select_rss_b,select_total_wall_s,select_total_rss_b"
+        ;;
+    *)
+        echo "ERROR: unknown stats type '${TYPE}'" >&2
+        exit 1
+        ;;
+esac
+
+# Find most recent existing run CSV (empty string if none).
+latest_csv=$(find "${STATS_DIR}" -maxdepth 1 -name 'run_*.csv' 2>/dev/null | sort | tail -1)
+
+# Check if any .stats file is newer than the latest run CSV.
+if [[ -n "${latest_csv}" ]] && \
+   [[ -z "$(find "${STATS_DIR}" -maxdepth 1 -name '*.stats' -newer "${latest_csv}" 2>/dev/null)" ]]; then
+    echo "[${TYPE}] stats up to date (${latest_csv})"
+    exit 0
+fi
+
+run_n=$(printf '%03d' "$(find "${STATS_DIR}" -maxdepth 1 -name 'run_*.csv' 2>/dev/null | wc -l | tr -d ' ')")
+CSV="${STATS_DIR}/run_${run_n}.csv"
+
+echo "${HEADER}" >"${CSV}"
+
+# Sort .stats files by name for reproducible row order.
+while IFS= read -r stats_file; do
+    sed "s/^/${run_n},/" "${stats_file}"
+done < <(find "${STATS_DIR}" -maxdepth 1 -name '*.stats' | sort) >>"${CSV}"
+
+echo "[${TYPE}] run ${run_n} → ${CSV}"

benchmark/build_reference.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""Build a reference kmer index from paired-end FASTQ reads.
+
+Extracts canonical kmers — min(kmer, revcomp(kmer)) encoded as uint64 —
+counts their abundances, and saves a sorted numpy pair (kmers, counts).
+
+Output .npz arrays
+  kmers  : uint64, sorted ascending — canonical kmer integers
+  counts : uint32, same order      — raw read abundances
+"""
+import argparse
+import gzip
+import sys
+from collections import defaultdict
+
+import numpy as np
+
+
+# ── encoding ────────────────────────────────────────────────────────────────
+
+_ENCODE = {'A': 0, 'C': 1, 'G': 2, 'T': 3,
+           'a': 0, 'c': 1, 'g': 2, 't': 3}
+
+# Lookup table: revcomp of one byte (4 bases, 8 bits).
+# Precomputed once at import time.
+_REVCOMP8 = [0] * 256
+for _i in range(256):
+    _rc, _x = 0, _i
+    for _ in range(4):
+        _rc = (_rc << 2) | (3 - (_x & 3))
+        _x >>= 2
+    _REVCOMP8[_i] = _rc
+del _i, _rc, _x
+
+
+def revcomp_int(kmer: int, k: int) -> int:
+    """Reverse-complement of a kmer encoded as an integer (2 bits/base).
+
+    Uses byte-level lookup (4 bases at a time) for speed.
+    """
+    rc = 0
+    bits_left = 2 * k
+    while bits_left > 0:
+        chunk = min(8, bits_left)
+        rc_byte = _REVCOMP8[kmer & 0xFF] >> (8 - chunk)
+        rc = (rc << chunk) | rc_byte
+        kmer >>= chunk
+        bits_left -= chunk
+    return rc
+
+
+# ── FASTQ parsing ────────────────────────────────────────────────────────────
+
+def iter_sequences(path: str):
+    """Yield raw sequences from a (gzipped) FASTQ file."""
+    opener = gzip.open if path.endswith('.gz') else open
+    with opener(path, 'rt') as fh:
+        while True:
+            if not fh.readline():   # '@' header
+                break
+            seq = fh.readline().rstrip('\n')
+            fh.readline()           # '+'
+            fh.readline()           # quality
+            yield seq
+
+
+# ── kmer counting ────────────────────────────────────────────────────────────
+
+def count_kmers(paths: list[str], k: int) -> dict[int, int]:
+    mask = (1 << (2 * k)) - 1
+    counts: dict[int, int] = defaultdict(int)
+    n_reads = 0
+
+    for path in paths:
+        for seq in iter_sequences(path):
+            n_reads += 1
+            kmer = 0
+            run = 0          # consecutive valid bases
+
+            for c in seq:
+                b = _ENCODE.get(c)
+                if b is None:    # N or unexpected character → reset
+                    kmer = 0
+                    run = 0
+                    continue
+                kmer = ((kmer << 2) | b) & mask
+                run += 1
+                if run >= k:
+                    rc = revcomp_int(kmer, k)
+                    counts[kmer if kmer <= rc else rc] += 1
+
+            if n_reads % 100_000 == 0:
+                print(f'  {n_reads:,} reads processed, '
+                      f'{len(counts):,} distinct kmers so far',
+                      file=sys.stderr)
+
+    print(f'  {n_reads:,} reads total, {len(counts):,} distinct kmers',
+          file=sys.stderr)
+    return counts
+
+
+# ── main ─────────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument('reads', nargs='+', metavar='FASTQ',
+                    help='Input reads (FASTQ, gzip OK)')
+    ap.add_argument('-k', '--kmer-size', type=int, default=31,
+                    metavar='K')
+    ap.add_argument('--min-abundance', type=int, default=1,
+                    metavar='N', help='Drop kmers with count < N (default 1)')
+    ap.add_argument('-o', '--output', required=True,
+                    metavar='FILE', help='Output .npz path')
+    args = ap.parse_args()
+
+    print(f'k={args.kmer_size}  files={len(args.reads)}', file=sys.stderr)
+    counts = count_kmers(args.reads, args.kmer_size)
+
+    if args.min_abundance > 1:
+        before = len(counts)
+        counts = {k: v for k, v in counts.items() if v >= args.min_abundance}
+        print(f'  min-abundance={args.min_abundance}: '
+              f'{before - len(counts):,} kmers dropped, '
+              f'{len(counts):,} retained',
+              file=sys.stderr)
+
+    print(f'Sorting and saving → {args.output}', file=sys.stderr)
+    kmers_arr  = np.fromiter(sorted(counts), dtype=np.uint64, count=len(counts))
+    counts_arr = np.array([counts[int(k)] for k in kmers_arr], dtype=np.uint32)
+
+    np.savez_compressed(args.output, kmers=kmers_arr, counts=counts_arr)
+    print(f'Done  {len(kmers_arr):,} kmers  →  {args.output}', file=sys.stderr)
+
+
+if __name__ == '__main__':
+    main()

benchmark/build_reference.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SIMDATA_DIR="${SCRIPT_DIR}/simulated_data"
+REF_DIR="${SCRIPT_DIR}/reference_index"
+PYTHON="${SCRIPT_DIR}/../.venv/bin/python3"
+BUILD_PY="${SCRIPT_DIR}/build_reference.py"
+
+KMER_SIZE="${KMER_SIZE:-31}"
+MIN_ABUNDANCE="${MIN_ABUNDANCE:-1}"
+
+mkdir -p "${REF_DIR}"
+
+for species_dir in "${SIMDATA_DIR}"/*/; do
+    [[ -d "${species_dir}" ]] || continue
+    species=$(basename "${species_dir}")
+
+    for strain_dir in "${species_dir}"*/; do
+        [[ -d "${strain_dir}" ]] || continue
+        strain=$(basename "${strain_dir}")
+
+        r1="${strain_dir}/reads_R1.fastq.gz"
+        r2="${strain_dir}/reads_R2.fastq.gz"
+        if [[ ! -f "${r1}" || ! -f "${r2}" ]]; then
+            echo "SKIP ${species}--${strain}: reads not found" >&2
+            continue
+        fi
+
+        out="${REF_DIR}/${species}--${strain}.npz"
+        echo "[${species}--${strain}] → ${out}"
+
+        "${PYTHON}" "${BUILD_PY}" \
+            --kmer-size      "${KMER_SIZE}" \
+            --min-abundance  "${MIN_ABUNDANCE}" \
+            --output         "${out}" \
+            "${r1}" "${r2}"
+    done
+done

benchmark/build_reference_dist.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+"""Compute reference pairwise distance matrices from per-specimen .npz kmer indexes.
+
+Reads all .npz files in reference_index/ (each containing sorted uint64 `kmers`
+and uint32 `counts`), computes all distance metrics supported by `obikmer distance`,
+and writes one CSV per metric to reference_dist/.
+
+Output CSV format matches `obikmer distance --output`:
+  - first row: "genome", then specimen names
+  - subsequent rows: specimen name, then float or int values
+
+Metrics written
+  jaccard_dist.csv          Jaccard distance (presence/absence)
+  shared_kmers.csv          Shared-kmer count matrix (intersection size)
+  bray_curtis_dist.csv      Bray-Curtis dissimilarity (raw counts)
+  relfreq_bray_curtis_dist.csv  Bray-Curtis on relative frequencies
+  euclidean_dist.csv        Euclidean distance (raw counts)
+  relfreq_euclidean_dist.csv    Euclidean distance on relative frequencies
+  hellinger_dist.csv        Hellinger distance
+  hellinger_euclidean_dist.csv  Euclidean distance in Hellinger space
+"""
+import argparse
+import sys
+from pathlib import Path
+
+import numpy as np
+
+
+# ── pairwise helpers ──────────────────────────────────────────────────────────
+
+def shared_indices(a_kmers: np.ndarray, b_kmers: np.ndarray):
+    """Return index arrays (idx_a, idx_b) for kmers present in both sets.
+
+    Both arrays must be sorted uint64. Uses searchsorted: O(|B| log |A|).
+    """
+    pos = np.searchsorted(a_kmers, b_kmers)
+    pos = np.clip(pos, 0, len(a_kmers) - 1)
+    mask = a_kmers[pos] == b_kmers
+    idx_b = np.where(mask)[0]
+    idx_a = pos[idx_b]
+    return idx_a, idx_b
+
+
+def pairwise_stats(specimens: list[dict]) -> dict[str, np.ndarray]:
+    """Compute all pairwise distance matrices at once.
+
+    Returns a dict metric_name → ndarray (n×n float64 or int64).
+    Each specimen dict has keys: name, kmers, counts.
+    """
+    n = len(specimens)
+
+    # Pre-compute per-specimen scalars
+    kmer_counts = np.array([len(s['kmers'])  for s in specimens], dtype=np.uint64)
+    count_sums  = np.array([s['counts'].sum() for s in specimens], dtype=np.uint64)
+
+    # Per-specimen sum-of-squares (for Euclidean decomposition)
+    sq_sums     = np.array([(s['counts'].astype(np.float64) ** 2).sum() for s in specimens])
+
+    # Allocate output matrices
+    shared_mat     = np.zeros((n, n), dtype=np.uint64)
+    hamming_mat    = np.zeros((n, n), dtype=np.float64)
+    jaccard_mat    = np.zeros((n, n), dtype=np.float64)
+    bray_mat       = np.zeros((n, n), dtype=np.float64)
+    relfreq_bray   = np.zeros((n, n), dtype=np.float64)
+    euclidean_mat  = np.zeros((n, n), dtype=np.float64)
+    relfreq_eucl   = np.zeros((n, n), dtype=np.float64)
+    hellinger_mat  = np.zeros((n, n), dtype=np.float64)
+    hell_eucl_mat  = np.zeros((n, n), dtype=np.float64)
+
+    for i in range(n):
+        a_km = specimens[i]['kmers']
+        a_ct = specimens[i]['counts'].astype(np.float64)
+        sa   = float(count_sums[i])
+        na   = int(kmer_counts[i])
+
+        for j in range(i + 1, n):
+            b_km = specimens[j]['kmers']
+            b_ct = specimens[j]['counts'].astype(np.float64)
+            sb   = float(count_sums[j])
+            nb   = int(kmer_counts[j])
+
+            idx_a, idx_b = shared_indices(a_km, b_km)
+            inter = len(idx_a)
+
+            ca_sh = a_ct[idx_a]
+            cb_sh = b_ct[idx_b]
+
+            # ── Presence metrics ──────────────────────────────────────────────
+
+            union   = na + nb - inter
+            jac     = (1.0 - inter / union) if union else 0.0
+            hamming = float(na + nb - 2 * inter)  # |A Δ B|
+
+            # ── Count metrics ─────────────────────────────────────────────────
+
+            # Bray-Curtis: 1 - 2*Σmin(a,b) / (Σa + Σb)
+            sum_min = np.minimum(ca_sh, cb_sh).sum()
+            denom_bc = sa + sb
+            bc = (1.0 - 2.0 * sum_min / denom_bc) if denom_bc else 0.0
+
+            # RelfreqBray: 1 - Σmin(a/sa, b/sb)  [only shared contribute]
+            if sa and sb:
+                rfb = 1.0 - np.minimum(ca_sh / sa, cb_sh / sb).sum()
+            else:
+                rfb = 0.0
+
+            # Euclidean: √(Σa² + Σb² - 2·Σ(a·b)_shared)
+            cross = (ca_sh * cb_sh).sum()
+            eucl_partial = sq_sums[i] + sq_sums[j] - 2.0 * cross
+            eucl = np.sqrt(max(eucl_partial, 0.0))
+
+            # RelfreqEuclidean: √(Σ(a/sa - b/sb)²)
+            # = √(Σa²/sa² + Σb²/sb² - 2·Σ(a·b)_shared/(sa·sb))
+            if sa and sb:
+                rf_cross = (ca_sh / sa * (cb_sh / sb)).sum()
+                rfe_partial = (sq_sums[i] / sa**2
+                               + sq_sums[j] / sb**2
+                               - 2.0 * rf_cross)
+                rfe = np.sqrt(max(rfe_partial, 0.0))
+            else:
+                rfe = 0.0
+
+            # Hellinger partial: Σ(√(a/sa) - √(b/sb))² over global universe
+            # = 2 - 2·Σ√(a·b)_shared / √(sa·sb)
+            if sa and sb:
+                bc_coeff = np.sqrt(ca_sh * cb_sh).sum() / np.sqrt(sa * sb)
+                hell_partial = max(2.0 - 2.0 * bc_coeff, 0.0)
+            else:
+                hell_partial = 0.0
+
+            sq2 = np.sqrt(2.0)
+            hell     = np.sqrt(hell_partial) / sq2
+            hell_euc = np.sqrt(hell_partial)
+
+            # ── Fill symmetric matrices ───────────────────────────────────────
+            for mat, val in [
+                (shared_mat,    inter),
+                (hamming_mat,   hamming),
+                (jaccard_mat,   jac),
+                (bray_mat,      bc),
+                (relfreq_bray,  rfb),
+                (euclidean_mat, eucl),
+                (relfreq_eucl,  rfe),
+                (hellinger_mat, hell),
+                (hell_eucl_mat, hell_euc),
+            ]:
+                mat[i, j] = val
+                mat[j, i] = val
+
+    return {
+        'shared_kmers':               shared_mat,
+        'hamming_dist':               hamming_mat,
+        'jaccard_dist':               jaccard_mat,
+        'bray_curtis_dist':           bray_mat,
+        'relfreq_bray_curtis_dist':   relfreq_bray,
+        'euclidean_dist':             euclidean_mat,
+        'relfreq_euclidean_dist':     relfreq_eucl,
+        'hellinger_dist':             hellinger_mat,
+        'hellinger_euclidean_dist':   hell_eucl_mat,
+    }
+
+
+# ── I/O ───────────────────────────────────────────────────────────────────────
+
+def write_csv(path: Path, labels: list[str], mat: np.ndarray, fmt: str) -> None:
+    with path.open('w') as fh:
+        fh.write('genome,' + ','.join(labels) + '\n')
+        for i, label in enumerate(labels):
+            row = ','.join(format(mat[i, j], fmt) for j in range(len(labels)))
+            fh.write(f'{label},{row}\n')
+    print(f'  → {path}', file=sys.stderr)
+
+
+# ── main ─────────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument('--ref-dir', default='reference_index',
+                    help='Directory with per-specimen .npz files (default: reference_index)')
+    ap.add_argument('--out-dir', default='reference_dist',
+                    help='Output directory for CSV files (default: reference_dist)')
+    args = ap.parse_args()
+
+    ref_dir = Path(args.ref_dir)
+    out_dir = Path(args.out_dir)
+    out_dir.mkdir(exist_ok=True)
+
+    npz_files = sorted(ref_dir.glob('*.npz'))
+    if not npz_files:
+        print(f'ERROR: no .npz files found in {ref_dir}', file=sys.stderr)
+        sys.exit(1)
+
+    print(f'Loading {len(npz_files)} specimen(s) from {ref_dir}/', file=sys.stderr)
+    specimens = []
+    for f in npz_files:
+        data = np.load(f)
+        specimens.append({
+            'name':   f.stem,
+            'kmers':  data['kmers'],
+            'counts': data['counts'],
+        })
+        print(f'  {f.stem}: {len(data["kmers"]):,} kmers', file=sys.stderr)
+
+    labels = [s['name'] for s in specimens]
+    n = len(labels)
+    print(f'\nComputing pairwise distances for {n} specimens…', file=sys.stderr)
+
+    matrices = pairwise_stats(specimens)
+
+    print(f'\nWriting CSVs to {out_dir}/', file=sys.stderr)
+    write_csv(out_dir / 'shared_kmers.csv',             labels, matrices['shared_kmers'],             'd')
+    write_csv(out_dir / 'hamming_dist.csv',             labels, matrices['hamming_dist'],             '.6f')
+    write_csv(out_dir / 'jaccard_dist.csv',             labels, matrices['jaccard_dist'],             '.6f')
+    write_csv(out_dir / 'bray_curtis_dist.csv',         labels, matrices['bray_curtis_dist'],         '.6f')
+    write_csv(out_dir / 'relfreq_bray_curtis_dist.csv', labels, matrices['relfreq_bray_curtis_dist'], '.6f')
+    write_csv(out_dir / 'euclidean_dist.csv',           labels, matrices['euclidean_dist'],           '.6f')
+    write_csv(out_dir / 'relfreq_euclidean_dist.csv',   labels, matrices['relfreq_euclidean_dist'],   '.6f')
+    write_csv(out_dir / 'hellinger_dist.csv',           labels, matrices['hellinger_dist'],           '.6f')
+    write_csv(out_dir / 'hellinger_euclidean_dist.csv', labels, matrices['hellinger_euclidean_dist'], '.6f')
+
+    print('\nDone.', file=sys.stderr)
+
+
+if __name__ == '__main__':
+    main()

benchmark/compare_all_dist.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""Compare all reference distance matrices against obikmer distance outputs.
+
+Reads from:
+  reference_dist/   — ground-truth matrices computed by build_reference_dist.py
+  obikmer_dist/     — matrices produced by `obikmer distance`
+
+Handles label reordering: both matrices are sorted by genome label before
+element-wise comparison, so column/row order differences are irrelevant.
+
+Output: stats/dist_comparison/summary.csv
+  comparison,max_abs,mean_abs,rmse,n_pairs,status
+"""
+import csv
+import sys
+from pathlib import Path
+
+import numpy as np
+
+
+# ── CSV loading ───────────────────────────────────────────────────────────────
+
+def load_matrix(path: Path) -> tuple[list[str], np.ndarray]:
+    """Load a distance-matrix CSV; return (sorted_labels, matrix_float64)."""
+    with path.open() as fh:
+        reader = csv.reader(fh)
+        header = next(reader)[1:]     # skip 'genome' column
+        raw: dict[str, list[float]] = {}
+        for row in reader:
+            raw[row[0]] = [float(x) for x in row[1:]]
+
+    label_to_col = {h: i for i, h in enumerate(header)}
+    labels = sorted(raw.keys())
+    n = len(labels)
+    mat = np.zeros((n, n), dtype=np.float64)
+    for i, ri in enumerate(labels):
+        for j, cj in enumerate(labels):
+            mat[i, j] = raw[ri][label_to_col[cj]]
+    return labels, mat
+
+
+# ── comparison ────────────────────────────────────────────────────────────────
+
+def compare(label: str,
+            ref_path: Path,
+            obi_path: Path,
+            tol: float = 1e-4) -> dict:
+    if not ref_path.exists():
+        return {'comparison': label, 'status': 'REF_MISSING',
+                'max_abs': '', 'mean_abs': '', 'rmse': '', 'n_pairs': ''}
+    if not obi_path.exists():
+        return {'comparison': label, 'status': 'OBI_MISSING',
+                'max_abs': '', 'mean_abs': '', 'rmse': '', 'n_pairs': ''}
+
+    ref_labels, ref_mat = load_matrix(ref_path)
+    obi_labels, obi_mat = load_matrix(obi_path)
+
+    if ref_labels != obi_labels:
+        only_ref = sorted(set(ref_labels) - set(obi_labels))
+        only_obi = sorted(set(obi_labels) - set(ref_labels))
+        print(f'  [{label}] label mismatch — '
+              f'only_ref={only_ref}  only_obi={only_obi}', file=sys.stderr)
+        return {'comparison': label, 'status': 'LABEL_MISMATCH',
+                'max_abs': '', 'mean_abs': '', 'rmse': '', 'n_pairs': ''}
+
+    n = len(ref_labels)
+    # Off-diagonal mask
+    mask = ~np.eye(n, dtype=bool)
+    diff = np.abs(ref_mat[mask] - obi_mat[mask])
+    n_pairs = diff.size
+
+    max_abs  = float(diff.max())
+    mean_abs = float(diff.mean())
+    rmse     = float(np.sqrt((diff ** 2).mean()))
+    status   = 'PASS' if max_abs <= tol else 'FAIL'
+
+    print(f'  [{label}] n={n_pairs}  '
+          f'max={max_abs:.3e}  mean={mean_abs:.3e}  rmse={rmse:.3e}  {status}',
+          file=sys.stderr)
+    return {
+        'comparison': label,
+        'max_abs':    f'{max_abs:.6e}',
+        'mean_abs':   f'{mean_abs:.6e}',
+        'rmse':       f'{rmse:.6e}',
+        'n_pairs':    str(n_pairs),
+        'status':     status,
+    }
+
+
+# ── comparison table ──────────────────────────────────────────────────────────
+
+# (label, ref_csv, obikmer_csv)
+# The reference jaccard/shared is presence-based, which should match both
+# presence/jaccard and count/jaccard (threshold=1).
+COMPARISONS = [
+    # ── presence index ────────────────────────────────────────────────────────
+    ('presence/jaccard_dist',
+     'reference_dist/jaccard_dist.csv',
+     'obikmer_dist/presence/jaccard_dist.csv'),
+
+    ('presence/jaccard_shared',
+     'reference_dist/shared_kmers.csv',
+     'obikmer_dist/presence/jaccard_shared.csv'),
+
+    ('presence/hamming_dist',
+     'reference_dist/hamming_dist.csv',
+     'obikmer_dist/presence/hamming_dist.csv'),
+
+    # ── count index (jaccard cross-check) ─────────────────────────────────────
+    ('count/jaccard_dist',
+     'reference_dist/jaccard_dist.csv',
+     'obikmer_dist/count/jaccard_dist.csv'),
+
+    ('count/jaccard_shared',
+     'reference_dist/shared_kmers.csv',
+     'obikmer_dist/count/jaccard_shared.csv'),
+
+    # ── count index (count-based metrics) ────────────────────────────────────
+    ('count/bray_curtis_dist',
+     'reference_dist/bray_curtis_dist.csv',
+     'obikmer_dist/count/bray_curtis_dist.csv'),
+
+    ('count/relfreq_bray_curtis_dist',
+     'reference_dist/relfreq_bray_curtis_dist.csv',
+     'obikmer_dist/count/relfreq_bray_curtis_dist.csv'),
+
+    ('count/euclidean_dist',
+     'reference_dist/euclidean_dist.csv',
+     'obikmer_dist/count/euclidean_dist.csv'),
+
+    ('count/relfreq_euclidean_dist',
+     'reference_dist/relfreq_euclidean_dist.csv',
+     'obikmer_dist/count/relfreq_euclidean_dist.csv'),
+
+    ('count/hellinger_dist',
+     'reference_dist/hellinger_dist.csv',
+     'obikmer_dist/count/hellinger_dist.csv'),
+
+    ('count/hellinger_euclidean_dist',
+     'reference_dist/hellinger_euclidean_dist.csv',
+     'obikmer_dist/count/hellinger_euclidean_dist.csv'),
+]
+
+
+# ── main ─────────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    import argparse
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument('--tol', type=float, default=1e-4,
+                    help='Max abs diff threshold for PASS/FAIL (default 1e-4)')
+    ap.add_argument('--out', default='stats/dist_comparison/summary.csv',
+                    help='Output summary CSV path')
+    args = ap.parse_args()
+
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    print(f'Comparing {len(COMPARISONS)} matrix pairs…', file=sys.stderr)
+    rows = []
+    for label, ref, obi in COMPARISONS:
+        rows.append(compare(label, Path(ref), Path(obi), tol=args.tol))
+
+    fields = ['comparison', 'max_abs', 'mean_abs', 'rmse', 'n_pairs', 'status']
+    with out_path.open('w', newline='') as fh:
+        w = csv.DictWriter(fh, fieldnames=fields)
+        w.writeheader()
+        w.writerows(rows)
+
+    print(f'\n→ {out_path}', file=sys.stderr)
+
+    n_fail = sum(1 for r in rows if r.get('status') == 'FAIL')
+    n_pass = sum(1 for r in rows if r.get('status') == 'PASS')
+    print(f'Summary: {n_pass} PASS  {n_fail} FAIL  '
+          f'{len(rows) - n_pass - n_fail} SKIP', file=sys.stderr)
+    if n_fail:
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

benchmark/deps.mk

@@ -0,0 +1,199 @@
+SPECIMENS := Escherichia_coli--K-12_MG1655 Escherichia_coli--EDL933 Salmonella_enterica--LT2 Escherichia_coli--CFT073 Bacillus_subtilis--168 Salmonella_enterica--P125109 Shouchella_clausii--KSM-K16 Escherichia_coli--K-12_W3110 Klebsiella_pneumoniae--MGH_78578 Opitutus_terrae--PB90-1 Saccharolobus_islandicus--M.16.4 Acidobacterium_capsulatum--ATCC_51196 Salmonella_enterica--AKU_12601 Proteus_mirabilis--HI4320 Salmonella_enterica--CT18 Klebsiella_pneumoniae--HS11286 Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1 Klebsiella_pneumoniae--ATCC_13883 Yersinia_ruckeri--YRB Candidozyma_auris--GCF_003013715.1_ASM301371v2
+SPECIES   := Escherichia_coli Salmonella_enterica Bacillus_subtilis Shouchella_clausii Klebsiella_pneumoniae Opitutus_terrae Saccharolobus_islandicus Acidobacterium_capsulatum Proteus_mirabilis Wolbachia_endosymbiont Yersinia_ruckeri Candidozyma_auris
+
+# Escherichia_coli--K-12_MG1655
+simulated_data/Escherichia_coli/K-12_MG1655/reads_R1.fastq.gz: genomes/GCF_000005845.2_ASM584v2_genomic.fna.gz
+reference_index/Escherichia_coli--K-12_MG1655.npz: simulated_data/Escherichia_coli/K-12_MG1655/reads_R1.fastq.gz
+specimen_index_presence/Escherichia_coli--K-12_MG1655/index.done stats/indexing_presence/Escherichia_coli--K-12_MG1655.stats: simulated_data/Escherichia_coli/K-12_MG1655/reads_R1.fastq.gz
+specimen_index_count/Escherichia_coli--K-12_MG1655/index.done stats/indexing_count/Escherichia_coli--K-12_MG1655.stats: simulated_data/Escherichia_coli/K-12_MG1655/reads_R1.fastq.gz
+stats/verify_presence/Escherichia_coli--K-12_MG1655.stats: reference_index/Escherichia_coli--K-12_MG1655.npz specimen_index_presence/Escherichia_coli--K-12_MG1655/index.done
+stats/verify_count/Escherichia_coli--K-12_MG1655.stats: reference_index/Escherichia_coli--K-12_MG1655.npz specimen_index_count/Escherichia_coli--K-12_MG1655/index.done
+
+# Escherichia_coli--EDL933
+simulated_data/Escherichia_coli/EDL933/reads_R1.fastq.gz: genomes/GCF_000006665.1_ASM666v1_genomic.fna.gz
+reference_index/Escherichia_coli--EDL933.npz: simulated_data/Escherichia_coli/EDL933/reads_R1.fastq.gz
+specimen_index_presence/Escherichia_coli--EDL933/index.done stats/indexing_presence/Escherichia_coli--EDL933.stats: simulated_data/Escherichia_coli/EDL933/reads_R1.fastq.gz
+specimen_index_count/Escherichia_coli--EDL933/index.done stats/indexing_count/Escherichia_coli--EDL933.stats: simulated_data/Escherichia_coli/EDL933/reads_R1.fastq.gz
+stats/verify_presence/Escherichia_coli--EDL933.stats: reference_index/Escherichia_coli--EDL933.npz specimen_index_presence/Escherichia_coli--EDL933/index.done
+stats/verify_count/Escherichia_coli--EDL933.stats: reference_index/Escherichia_coli--EDL933.npz specimen_index_count/Escherichia_coli--EDL933/index.done
+
+# Salmonella_enterica--LT2
+simulated_data/Salmonella_enterica/LT2/reads_R1.fastq.gz: genomes/GCF_000006945.2_ASM694v2_genomic.fna.gz
+reference_index/Salmonella_enterica--LT2.npz: simulated_data/Salmonella_enterica/LT2/reads_R1.fastq.gz
+specimen_index_presence/Salmonella_enterica--LT2/index.done stats/indexing_presence/Salmonella_enterica--LT2.stats: simulated_data/Salmonella_enterica/LT2/reads_R1.fastq.gz
+specimen_index_count/Salmonella_enterica--LT2/index.done stats/indexing_count/Salmonella_enterica--LT2.stats: simulated_data/Salmonella_enterica/LT2/reads_R1.fastq.gz
+stats/verify_presence/Salmonella_enterica--LT2.stats: reference_index/Salmonella_enterica--LT2.npz specimen_index_presence/Salmonella_enterica--LT2/index.done
+stats/verify_count/Salmonella_enterica--LT2.stats: reference_index/Salmonella_enterica--LT2.npz specimen_index_count/Salmonella_enterica--LT2/index.done
+
+# Escherichia_coli--CFT073
+simulated_data/Escherichia_coli/CFT073/reads_R1.fastq.gz: genomes/GCF_000007445.1_ASM744v1_genomic.fna.gz
+reference_index/Escherichia_coli--CFT073.npz: simulated_data/Escherichia_coli/CFT073/reads_R1.fastq.gz
+specimen_index_presence/Escherichia_coli--CFT073/index.done stats/indexing_presence/Escherichia_coli--CFT073.stats: simulated_data/Escherichia_coli/CFT073/reads_R1.fastq.gz
+specimen_index_count/Escherichia_coli--CFT073/index.done stats/indexing_count/Escherichia_coli--CFT073.stats: simulated_data/Escherichia_coli/CFT073/reads_R1.fastq.gz
+stats/verify_presence/Escherichia_coli--CFT073.stats: reference_index/Escherichia_coli--CFT073.npz specimen_index_presence/Escherichia_coli--CFT073/index.done
+stats/verify_count/Escherichia_coli--CFT073.stats: reference_index/Escherichia_coli--CFT073.npz specimen_index_count/Escherichia_coli--CFT073/index.done
+
+# Bacillus_subtilis--168
+simulated_data/Bacillus_subtilis/168/reads_R1.fastq.gz: genomes/GCF_000009045.1_ASM904v1_genomic.fna.gz
+reference_index/Bacillus_subtilis--168.npz: simulated_data/Bacillus_subtilis/168/reads_R1.fastq.gz
+specimen_index_presence/Bacillus_subtilis--168/index.done stats/indexing_presence/Bacillus_subtilis--168.stats: simulated_data/Bacillus_subtilis/168/reads_R1.fastq.gz
+specimen_index_count/Bacillus_subtilis--168/index.done stats/indexing_count/Bacillus_subtilis--168.stats: simulated_data/Bacillus_subtilis/168/reads_R1.fastq.gz
+stats/verify_presence/Bacillus_subtilis--168.stats: reference_index/Bacillus_subtilis--168.npz specimen_index_presence/Bacillus_subtilis--168/index.done
+stats/verify_count/Bacillus_subtilis--168.stats: reference_index/Bacillus_subtilis--168.npz specimen_index_count/Bacillus_subtilis--168/index.done
+
+# Salmonella_enterica--P125109
+simulated_data/Salmonella_enterica/P125109/reads_R1.fastq.gz: genomes/GCF_000009505.1_ASM950v1_genomic.fna.gz
+reference_index/Salmonella_enterica--P125109.npz: simulated_data/Salmonella_enterica/P125109/reads_R1.fastq.gz
+specimen_index_presence/Salmonella_enterica--P125109/index.done stats/indexing_presence/Salmonella_enterica--P125109.stats: simulated_data/Salmonella_enterica/P125109/reads_R1.fastq.gz
+specimen_index_count/Salmonella_enterica--P125109/index.done stats/indexing_count/Salmonella_enterica--P125109.stats: simulated_data/Salmonella_enterica/P125109/reads_R1.fastq.gz
+stats/verify_presence/Salmonella_enterica--P125109.stats: reference_index/Salmonella_enterica--P125109.npz specimen_index_presence/Salmonella_enterica--P125109/index.done
+stats/verify_count/Salmonella_enterica--P125109.stats: reference_index/Salmonella_enterica--P125109.npz specimen_index_count/Salmonella_enterica--P125109/index.done
+
+# Shouchella_clausii--KSM-K16
+simulated_data/Shouchella_clausii/KSM-K16/reads_R1.fastq.gz: genomes/GCF_000009825.1_ASM982v1_genomic.fna.gz
+reference_index/Shouchella_clausii--KSM-K16.npz: simulated_data/Shouchella_clausii/KSM-K16/reads_R1.fastq.gz
+specimen_index_presence/Shouchella_clausii--KSM-K16/index.done stats/indexing_presence/Shouchella_clausii--KSM-K16.stats: simulated_data/Shouchella_clausii/KSM-K16/reads_R1.fastq.gz
+specimen_index_count/Shouchella_clausii--KSM-K16/index.done stats/indexing_count/Shouchella_clausii--KSM-K16.stats: simulated_data/Shouchella_clausii/KSM-K16/reads_R1.fastq.gz
+stats/verify_presence/Shouchella_clausii--KSM-K16.stats: reference_index/Shouchella_clausii--KSM-K16.npz specimen_index_presence/Shouchella_clausii--KSM-K16/index.done
+stats/verify_count/Shouchella_clausii--KSM-K16.stats: reference_index/Shouchella_clausii--KSM-K16.npz specimen_index_count/Shouchella_clausii--KSM-K16/index.done
+
+# Escherichia_coli--K-12_W3110
+simulated_data/Escherichia_coli/K-12_W3110/reads_R1.fastq.gz: genomes/GCF_000010245.2_ASM1024v1_genomic.fna.gz
+reference_index/Escherichia_coli--K-12_W3110.npz: simulated_data/Escherichia_coli/K-12_W3110/reads_R1.fastq.gz
+specimen_index_presence/Escherichia_coli--K-12_W3110/index.done stats/indexing_presence/Escherichia_coli--K-12_W3110.stats: simulated_data/Escherichia_coli/K-12_W3110/reads_R1.fastq.gz
+specimen_index_count/Escherichia_coli--K-12_W3110/index.done stats/indexing_count/Escherichia_coli--K-12_W3110.stats: simulated_data/Escherichia_coli/K-12_W3110/reads_R1.fastq.gz
+stats/verify_presence/Escherichia_coli--K-12_W3110.stats: reference_index/Escherichia_coli--K-12_W3110.npz specimen_index_presence/Escherichia_coli--K-12_W3110/index.done
+stats/verify_count/Escherichia_coli--K-12_W3110.stats: reference_index/Escherichia_coli--K-12_W3110.npz specimen_index_count/Escherichia_coli--K-12_W3110/index.done
+
+# Klebsiella_pneumoniae--MGH_78578
+simulated_data/Klebsiella_pneumoniae/MGH_78578/reads_R1.fastq.gz: genomes/GCF_000016305.1_ASM1630v1_genomic.fna.gz
+reference_index/Klebsiella_pneumoniae--MGH_78578.npz: simulated_data/Klebsiella_pneumoniae/MGH_78578/reads_R1.fastq.gz
+specimen_index_presence/Klebsiella_pneumoniae--MGH_78578/index.done stats/indexing_presence/Klebsiella_pneumoniae--MGH_78578.stats: simulated_data/Klebsiella_pneumoniae/MGH_78578/reads_R1.fastq.gz
+specimen_index_count/Klebsiella_pneumoniae--MGH_78578/index.done stats/indexing_count/Klebsiella_pneumoniae--MGH_78578.stats: simulated_data/Klebsiella_pneumoniae/MGH_78578/reads_R1.fastq.gz
+stats/verify_presence/Klebsiella_pneumoniae--MGH_78578.stats: reference_index/Klebsiella_pneumoniae--MGH_78578.npz specimen_index_presence/Klebsiella_pneumoniae--MGH_78578/index.done
+stats/verify_count/Klebsiella_pneumoniae--MGH_78578.stats: reference_index/Klebsiella_pneumoniae--MGH_78578.npz specimen_index_count/Klebsiella_pneumoniae--MGH_78578/index.done
+
+# Opitutus_terrae--PB90-1
+simulated_data/Opitutus_terrae/PB90-1/reads_R1.fastq.gz: genomes/GCF_000019965.1_ASM1996v1_genomic.fna.gz
+reference_index/Opitutus_terrae--PB90-1.npz: simulated_data/Opitutus_terrae/PB90-1/reads_R1.fastq.gz
+specimen_index_presence/Opitutus_terrae--PB90-1/index.done stats/indexing_presence/Opitutus_terrae--PB90-1.stats: simulated_data/Opitutus_terrae/PB90-1/reads_R1.fastq.gz
+specimen_index_count/Opitutus_terrae--PB90-1/index.done stats/indexing_count/Opitutus_terrae--PB90-1.stats: simulated_data/Opitutus_terrae/PB90-1/reads_R1.fastq.gz
+stats/verify_presence/Opitutus_terrae--PB90-1.stats: reference_index/Opitutus_terrae--PB90-1.npz specimen_index_presence/Opitutus_terrae--PB90-1/index.done
+stats/verify_count/Opitutus_terrae--PB90-1.stats: reference_index/Opitutus_terrae--PB90-1.npz specimen_index_count/Opitutus_terrae--PB90-1/index.done
+
+# Saccharolobus_islandicus--M.16.4
+simulated_data/Saccharolobus_islandicus/M.16.4/reads_R1.fastq.gz: genomes/GCF_000022445.1_ASM2244v1_genomic.fna.gz
+reference_index/Saccharolobus_islandicus--M.16.4.npz: simulated_data/Saccharolobus_islandicus/M.16.4/reads_R1.fastq.gz
+specimen_index_presence/Saccharolobus_islandicus--M.16.4/index.done stats/indexing_presence/Saccharolobus_islandicus--M.16.4.stats: simulated_data/Saccharolobus_islandicus/M.16.4/reads_R1.fastq.gz
+specimen_index_count/Saccharolobus_islandicus--M.16.4/index.done stats/indexing_count/Saccharolobus_islandicus--M.16.4.stats: simulated_data/Saccharolobus_islandicus/M.16.4/reads_R1.fastq.gz
+stats/verify_presence/Saccharolobus_islandicus--M.16.4.stats: reference_index/Saccharolobus_islandicus--M.16.4.npz specimen_index_presence/Saccharolobus_islandicus--M.16.4/index.done
+stats/verify_count/Saccharolobus_islandicus--M.16.4.stats: reference_index/Saccharolobus_islandicus--M.16.4.npz specimen_index_count/Saccharolobus_islandicus--M.16.4/index.done
+
+# Acidobacterium_capsulatum--ATCC_51196
+simulated_data/Acidobacterium_capsulatum/ATCC_51196/reads_R1.fastq.gz: genomes/GCF_000022565.1_ASM2256v1_genomic.fna.gz
+reference_index/Acidobacterium_capsulatum--ATCC_51196.npz: simulated_data/Acidobacterium_capsulatum/ATCC_51196/reads_R1.fastq.gz
+specimen_index_presence/Acidobacterium_capsulatum--ATCC_51196/index.done stats/indexing_presence/Acidobacterium_capsulatum--ATCC_51196.stats: simulated_data/Acidobacterium_capsulatum/ATCC_51196/reads_R1.fastq.gz
+specimen_index_count/Acidobacterium_capsulatum--ATCC_51196/index.done stats/indexing_count/Acidobacterium_capsulatum--ATCC_51196.stats: simulated_data/Acidobacterium_capsulatum/ATCC_51196/reads_R1.fastq.gz
+stats/verify_presence/Acidobacterium_capsulatum--ATCC_51196.stats: reference_index/Acidobacterium_capsulatum--ATCC_51196.npz specimen_index_presence/Acidobacterium_capsulatum--ATCC_51196/index.done
+stats/verify_count/Acidobacterium_capsulatum--ATCC_51196.stats: reference_index/Acidobacterium_capsulatum--ATCC_51196.npz specimen_index_count/Acidobacterium_capsulatum--ATCC_51196/index.done
+
+# Salmonella_enterica--AKU_12601
+simulated_data/Salmonella_enterica/AKU_12601/reads_R1.fastq.gz: genomes/GCF_000026565.1_ASM2656v1_genomic.fna.gz
+reference_index/Salmonella_enterica--AKU_12601.npz: simulated_data/Salmonella_enterica/AKU_12601/reads_R1.fastq.gz
+specimen_index_presence/Salmonella_enterica--AKU_12601/index.done stats/indexing_presence/Salmonella_enterica--AKU_12601.stats: simulated_data/Salmonella_enterica/AKU_12601/reads_R1.fastq.gz
+specimen_index_count/Salmonella_enterica--AKU_12601/index.done stats/indexing_count/Salmonella_enterica--AKU_12601.stats: simulated_data/Salmonella_enterica/AKU_12601/reads_R1.fastq.gz
+stats/verify_presence/Salmonella_enterica--AKU_12601.stats: reference_index/Salmonella_enterica--AKU_12601.npz specimen_index_presence/Salmonella_enterica--AKU_12601/index.done
+stats/verify_count/Salmonella_enterica--AKU_12601.stats: reference_index/Salmonella_enterica--AKU_12601.npz specimen_index_count/Salmonella_enterica--AKU_12601/index.done
+
+# Proteus_mirabilis--HI4320
+simulated_data/Proteus_mirabilis/HI4320/reads_R1.fastq.gz: genomes/GCF_000069965.1_ASM6996v1_genomic.fna.gz
+reference_index/Proteus_mirabilis--HI4320.npz: simulated_data/Proteus_mirabilis/HI4320/reads_R1.fastq.gz
+specimen_index_presence/Proteus_mirabilis--HI4320/index.done stats/indexing_presence/Proteus_mirabilis--HI4320.stats: simulated_data/Proteus_mirabilis/HI4320/reads_R1.fastq.gz
+specimen_index_count/Proteus_mirabilis--HI4320/index.done stats/indexing_count/Proteus_mirabilis--HI4320.stats: simulated_data/Proteus_mirabilis/HI4320/reads_R1.fastq.gz
+stats/verify_presence/Proteus_mirabilis--HI4320.stats: reference_index/Proteus_mirabilis--HI4320.npz specimen_index_presence/Proteus_mirabilis--HI4320/index.done
+stats/verify_count/Proteus_mirabilis--HI4320.stats: reference_index/Proteus_mirabilis--HI4320.npz specimen_index_count/Proteus_mirabilis--HI4320/index.done
+
+# Salmonella_enterica--CT18
+simulated_data/Salmonella_enterica/CT18/reads_R1.fastq.gz: genomes/GCF_000195995.1_ASM19599v1_genomic.fna.gz
+reference_index/Salmonella_enterica--CT18.npz: simulated_data/Salmonella_enterica/CT18/reads_R1.fastq.gz
+specimen_index_presence/Salmonella_enterica--CT18/index.done stats/indexing_presence/Salmonella_enterica--CT18.stats: simulated_data/Salmonella_enterica/CT18/reads_R1.fastq.gz
+specimen_index_count/Salmonella_enterica--CT18/index.done stats/indexing_count/Salmonella_enterica--CT18.stats: simulated_data/Salmonella_enterica/CT18/reads_R1.fastq.gz
+stats/verify_presence/Salmonella_enterica--CT18.stats: reference_index/Salmonella_enterica--CT18.npz specimen_index_presence/Salmonella_enterica--CT18/index.done
+stats/verify_count/Salmonella_enterica--CT18.stats: reference_index/Salmonella_enterica--CT18.npz specimen_index_count/Salmonella_enterica--CT18/index.done
+
+# Klebsiella_pneumoniae--HS11286
+simulated_data/Klebsiella_pneumoniae/HS11286/reads_R1.fastq.gz: genomes/GCF_000240185.1_ASM24018v2_genomic.fna.gz
+reference_index/Klebsiella_pneumoniae--HS11286.npz: simulated_data/Klebsiella_pneumoniae/HS11286/reads_R1.fastq.gz
+specimen_index_presence/Klebsiella_pneumoniae--HS11286/index.done stats/indexing_presence/Klebsiella_pneumoniae--HS11286.stats: simulated_data/Klebsiella_pneumoniae/HS11286/reads_R1.fastq.gz
+specimen_index_count/Klebsiella_pneumoniae--HS11286/index.done stats/indexing_count/Klebsiella_pneumoniae--HS11286.stats: simulated_data/Klebsiella_pneumoniae/HS11286/reads_R1.fastq.gz
+stats/verify_presence/Klebsiella_pneumoniae--HS11286.stats: reference_index/Klebsiella_pneumoniae--HS11286.npz specimen_index_presence/Klebsiella_pneumoniae--HS11286/index.done
+stats/verify_count/Klebsiella_pneumoniae--HS11286.stats: reference_index/Klebsiella_pneumoniae--HS11286.npz specimen_index_count/Klebsiella_pneumoniae--HS11286/index.done
+
+# Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1
+simulated_data/Wolbachia_endosymbiont/GCF_000306885.1_ASM30688v1/reads_R1.fastq.gz: genomes/GCF_000306885.1_ASM30688v1_genomic.fna.gz
+reference_index/Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1.npz: simulated_data/Wolbachia_endosymbiont/GCF_000306885.1_ASM30688v1/reads_R1.fastq.gz
+specimen_index_presence/Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1/index.done stats/indexing_presence/Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1.stats: simulated_data/Wolbachia_endosymbiont/GCF_000306885.1_ASM30688v1/reads_R1.fastq.gz
+specimen_index_count/Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1/index.done stats/indexing_count/Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1.stats: simulated_data/Wolbachia_endosymbiont/GCF_000306885.1_ASM30688v1/reads_R1.fastq.gz
+stats/verify_presence/Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1.stats: reference_index/Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1.npz specimen_index_presence/Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1/index.done
+stats/verify_count/Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1.stats: reference_index/Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1.npz specimen_index_count/Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1/index.done
+
+# Klebsiella_pneumoniae--ATCC_13883
+simulated_data/Klebsiella_pneumoniae/ATCC_13883/reads_R1.fastq.gz: genomes/GCF_000742135.1_ASM74213v1_genomic.fna.gz
+reference_index/Klebsiella_pneumoniae--ATCC_13883.npz: simulated_data/Klebsiella_pneumoniae/ATCC_13883/reads_R1.fastq.gz
+specimen_index_presence/Klebsiella_pneumoniae--ATCC_13883/index.done stats/indexing_presence/Klebsiella_pneumoniae--ATCC_13883.stats: simulated_data/Klebsiella_pneumoniae/ATCC_13883/reads_R1.fastq.gz
+specimen_index_count/Klebsiella_pneumoniae--ATCC_13883/index.done stats/indexing_count/Klebsiella_pneumoniae--ATCC_13883.stats: simulated_data/Klebsiella_pneumoniae/ATCC_13883/reads_R1.fastq.gz
+stats/verify_presence/Klebsiella_pneumoniae--ATCC_13883.stats: reference_index/Klebsiella_pneumoniae--ATCC_13883.npz specimen_index_presence/Klebsiella_pneumoniae--ATCC_13883/index.done
+stats/verify_count/Klebsiella_pneumoniae--ATCC_13883.stats: reference_index/Klebsiella_pneumoniae--ATCC_13883.npz specimen_index_count/Klebsiella_pneumoniae--ATCC_13883/index.done
+
+# Yersinia_ruckeri--YRB
+simulated_data/Yersinia_ruckeri/YRB/reads_R1.fastq.gz: genomes/GCF_000834255.1_ASM83425v1_genomic.fna.gz
+reference_index/Yersinia_ruckeri--YRB.npz: simulated_data/Yersinia_ruckeri/YRB/reads_R1.fastq.gz
+specimen_index_presence/Yersinia_ruckeri--YRB/index.done stats/indexing_presence/Yersinia_ruckeri--YRB.stats: simulated_data/Yersinia_ruckeri/YRB/reads_R1.fastq.gz
+specimen_index_count/Yersinia_ruckeri--YRB/index.done stats/indexing_count/Yersinia_ruckeri--YRB.stats: simulated_data/Yersinia_ruckeri/YRB/reads_R1.fastq.gz
+stats/verify_presence/Yersinia_ruckeri--YRB.stats: reference_index/Yersinia_ruckeri--YRB.npz specimen_index_presence/Yersinia_ruckeri--YRB/index.done
+stats/verify_count/Yersinia_ruckeri--YRB.stats: reference_index/Yersinia_ruckeri--YRB.npz specimen_index_count/Yersinia_ruckeri--YRB/index.done
+
+# Candidozyma_auris--GCF_003013715.1_ASM301371v2
+simulated_data/Candidozyma_auris/GCF_003013715.1_ASM301371v2/reads_R1.fastq.gz: genomes/GCF_003013715.1_ASM301371v2_genomic.fna.gz
+reference_index/Candidozyma_auris--GCF_003013715.1_ASM301371v2.npz: simulated_data/Candidozyma_auris/GCF_003013715.1_ASM301371v2/reads_R1.fastq.gz
+specimen_index_presence/Candidozyma_auris--GCF_003013715.1_ASM301371v2/index.done stats/indexing_presence/Candidozyma_auris--GCF_003013715.1_ASM301371v2.stats: simulated_data/Candidozyma_auris/GCF_003013715.1_ASM301371v2/reads_R1.fastq.gz
+specimen_index_count/Candidozyma_auris--GCF_003013715.1_ASM301371v2/index.done stats/indexing_count/Candidozyma_auris--GCF_003013715.1_ASM301371v2.stats: simulated_data/Candidozyma_auris/GCF_003013715.1_ASM301371v2/reads_R1.fastq.gz
+stats/verify_presence/Candidozyma_auris--GCF_003013715.1_ASM301371v2.stats: reference_index/Candidozyma_auris--GCF_003013715.1_ASM301371v2.npz specimen_index_presence/Candidozyma_auris--GCF_003013715.1_ASM301371v2/index.done
+stats/verify_count/Candidozyma_auris--GCF_003013715.1_ASM301371v2.stats: reference_index/Candidozyma_auris--GCF_003013715.1_ASM301371v2.npz specimen_index_count/Candidozyma_auris--GCF_003013715.1_ASM301371v2/index.done
+
+# Escherichia_coli
+specific_index_presence/Escherichia_coli/index.done stats/specific_kmer_presence/Escherichia_coli.stats: global_index_presence/index.done
+specific_index_count/Escherichia_coli/index.done stats/specific_kmer_count/Escherichia_coli.stats: global_index_count/index.done
+# Salmonella_enterica
+specific_index_presence/Salmonella_enterica/index.done stats/specific_kmer_presence/Salmonella_enterica.stats: global_index_presence/index.done
+specific_index_count/Salmonella_enterica/index.done stats/specific_kmer_count/Salmonella_enterica.stats: global_index_count/index.done
+# Bacillus_subtilis
+specific_index_presence/Bacillus_subtilis/index.done stats/specific_kmer_presence/Bacillus_subtilis.stats: global_index_presence/index.done
+specific_index_count/Bacillus_subtilis/index.done stats/specific_kmer_count/Bacillus_subtilis.stats: global_index_count/index.done
+# Shouchella_clausii
+specific_index_presence/Shouchella_clausii/index.done stats/specific_kmer_presence/Shouchella_clausii.stats: global_index_presence/index.done
+specific_index_count/Shouchella_clausii/index.done stats/specific_kmer_count/Shouchella_clausii.stats: global_index_count/index.done
+# Klebsiella_pneumoniae
+specific_index_presence/Klebsiella_pneumoniae/index.done stats/specific_kmer_presence/Klebsiella_pneumoniae.stats: global_index_presence/index.done
+specific_index_count/Klebsiella_pneumoniae/index.done stats/specific_kmer_count/Klebsiella_pneumoniae.stats: global_index_count/index.done
+# Opitutus_terrae
+specific_index_presence/Opitutus_terrae/index.done stats/specific_kmer_presence/Opitutus_terrae.stats: global_index_presence/index.done
+specific_index_count/Opitutus_terrae/index.done stats/specific_kmer_count/Opitutus_terrae.stats: global_index_count/index.done
+# Saccharolobus_islandicus
+specific_index_presence/Saccharolobus_islandicus/index.done stats/specific_kmer_presence/Saccharolobus_islandicus.stats: global_index_presence/index.done
+specific_index_count/Saccharolobus_islandicus/index.done stats/specific_kmer_count/Saccharolobus_islandicus.stats: global_index_count/index.done
+# Acidobacterium_capsulatum
+specific_index_presence/Acidobacterium_capsulatum/index.done stats/specific_kmer_presence/Acidobacterium_capsulatum.stats: global_index_presence/index.done
+specific_index_count/Acidobacterium_capsulatum/index.done stats/specific_kmer_count/Acidobacterium_capsulatum.stats: global_index_count/index.done
+# Proteus_mirabilis
+specific_index_presence/Proteus_mirabilis/index.done stats/specific_kmer_presence/Proteus_mirabilis.stats: global_index_presence/index.done
+specific_index_count/Proteus_mirabilis/index.done stats/specific_kmer_count/Proteus_mirabilis.stats: global_index_count/index.done
+# Wolbachia_endosymbiont
+specific_index_presence/Wolbachia_endosymbiont/index.done stats/specific_kmer_presence/Wolbachia_endosymbiont.stats: global_index_presence/index.done
+specific_index_count/Wolbachia_endosymbiont/index.done stats/specific_kmer_count/Wolbachia_endosymbiont.stats: global_index_count/index.done
+# Yersinia_ruckeri
+specific_index_presence/Yersinia_ruckeri/index.done stats/specific_kmer_presence/Yersinia_ruckeri.stats: global_index_presence/index.done
+specific_index_count/Yersinia_ruckeri/index.done stats/specific_kmer_count/Yersinia_ruckeri.stats: global_index_count/index.done
+# Candidozyma_auris
+specific_index_presence/Candidozyma_auris/index.done stats/specific_kmer_presence/Candidozyma_auris.stats: global_index_presence/index.done
+specific_index_count/Candidozyma_auris/index.done stats/specific_kmer_count/Candidozyma_auris.stats: global_index_count/index.done

benchmark/downloads.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+assemblies=(
+    GCF_000005845.2
+    GCF_000010245.2
+    GCF_000007445.1
+    GCF_000006665.1
+
+    GCF_000006945.2
+    GCF_000195995.1
+    GCF_000009505.1
+    GCF_000026565.1
+
+    GCF_000016305.1
+    GCF_000019965.1
+    GCF_000240185.1
+    GCF_000742135.1
+
+    GCF_000069965.1
+    GCF_000022565.1
+    GCF_000306885.1
+    GCF_003013715.1
+
+    GCF_000009045.1
+    GCF_000009825.1
+    GCF_000022445.1
+    GCF_000834255.1
+)
+
+mkdir -p genomes
+
+for acc in "${assemblies[@]}"; do
+    echo "Downloading ${acc}"
+
+    datasets download genome accession "${acc}" \
+        --include genome \
+        --filename "${acc}.zip"
+
+    unzip -q "${acc}.zip" -d "${acc}"
+    find "${acc}" -name "*.fna" |
+        while read file; do
+            obiconvert -Z ${file} >genomes/$(basename ${file}).gz
+        done
+
+    rm -rf "${acc}" "${acc}.zip"
+done

benchmark/filter_one_count.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Usage: filter_one_count.sh SPECIES
+# Filters global_index_count to keep only kmers specific to SPECIES,
+# then selects the SPECIES column in-place.
+# Outputs:
+#   specific_index_count/SPECIES/index.done  (written by obikmer select)
+#   stats/specific_kmer_count/SPECIES.stats  (one CSV data row, no header)
+set -euo pipefail
+
+SPECIES="$1"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
+
+SOURCE="${SCRIPT_DIR}/global_index_count"
+OUTPUT="${SCRIPT_DIR}/specific_index_count/${SPECIES}"
+STATS_DIR="${SCRIPT_DIR}/stats/specific_kmer_count"
+STATS_FILE="${STATS_DIR}/${SPECIES}.stats"
+
+mkdir -p "${STATS_DIR}"
+
+echo "[${SPECIES}] filter (count) → ${OUTPUT}"
+
+LOG_FILTER=$(mktemp)
+LOG_SELECT=$(mktemp)
+trap 'rm -f "${LOG_FILTER}" "${LOG_SELECT}"' EXIT
+
+"${BINARY}" filter \
+    --output "${OUTPUT}" \
+    --force \
+    --ingroup "species=${SPECIES}" \
+    --outgroup all \
+    --min-frac 0.5 \
+    --max-frac 1.0 \
+    --max-outgroup-count 0 \
+    "${SOURCE}" \
+    2>"${LOG_FILTER}"
+
+cat "${LOG_FILTER}" >&2
+
+"${BINARY}" select \
+    --in-place \
+    --group "${SPECIES}:species=${SPECIES}" \
+    --group-op "${SPECIES}:any" \
+    --select "${SPECIES}" \
+    "${OUTPUT}" \
+    2>"${LOG_SELECT}"
+
+cat "${LOG_SELECT}" >&2
+
+python3 - "${SPECIES}" "${LOG_FILTER}" "${LOG_SELECT}" <<'PYEOF' >"${STATS_FILE}"
+import sys, re
+
+species, log_filter, log_select = sys.argv[1], sys.argv[2], sys.argv[3]
+
+def strip_ansi(s):
+    return re.sub(r'\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]', '', s)
+
+def parse_wall(s):
+    s = s.strip()
+    if s.endswith('ms'): return float(s[:-2]) / 1000.0
+    if s.endswith('s'):  return float(s[:-1])
+    return 0.0
+
+def parse_rss(s):
+    m = re.match(r'([\d.]+)\s*(GB|MB|KB|B)', s.strip())
+    if not m: return 0
+    return int(float(m.group(1)) * {'GB': 1<<30, 'MB': 1<<20, 'KB': 1024, 'B': 1}[m.group(2)])
+
+def is_sep(s):
+    return bool(s) and not re.search(r'[A-Za-z0-9]', s)
+
+def parse_reporter(logfile):
+    stats = {}
+    state = 'scan'
+    with open(logfile, errors='replace') as fh:
+        for raw in fh:
+            line = strip_ansi(raw.rstrip('\n'))
+            s    = line.strip()
+            if state == 'scan':
+                if re.search(r'\bstage\b.*\bwall\b', line):
+                    state = 'in_header'
+            elif state == 'in_header':
+                if is_sep(s): state = 'rows'
+            elif state == 'rows':
+                if is_sep(s): state = 'total'
+                elif s:
+                    parts = re.split(r'  +', s)
+                    if len(parts) >= 4:
+                        stats[parts[0]] = (parse_wall(parts[1]), parse_rss(parts[3]))
+            elif state == 'total':
+                if s:
+                    parts = re.split(r'  +', s)
+                    if len(parts) >= 3:
+                        stats['TOTAL'] = (parse_wall(parts[1]),
+                                          parse_rss(parts[3]) if len(parts) > 3 else 0)
+                break
+    return stats
+
+f = parse_reporter(log_filter)
+s = parse_reporter(log_select)
+
+row = [species]
+for stage, d in [('rebuild', f), ('pack', f), ('filter_total', f), ('select', s), ('select_total', s)]:
+    key = 'TOTAL' if stage.endswith('_total') else stage
+    w, r = d.get(key, ('', ''))
+    row += [f'{w:.3f}' if isinstance(w, float) else '', str(r)]
+print(','.join(row))
+PYEOF

benchmark/filter_one_presence.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Usage: filter_one_presence.sh SPECIES
+# Filters global_index_presence to keep only kmers specific to SPECIES,
+# then selects the SPECIES column in-place.
+# Outputs:
+#   specific_index_presence/SPECIES/index.done  (written by obikmer select)
+#   stats/specific_kmer_presence/SPECIES.stats  (one CSV data row, no header)
+set -euo pipefail
+
+SPECIES="$1"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
+
+SOURCE="${SCRIPT_DIR}/global_index_presence"
+OUTPUT="${SCRIPT_DIR}/specific_index_presence/${SPECIES}"
+STATS_DIR="${SCRIPT_DIR}/stats/specific_kmer_presence"
+STATS_FILE="${STATS_DIR}/${SPECIES}.stats"
+
+mkdir -p "${STATS_DIR}"
+
+echo "[${SPECIES}] filter (presence) → ${OUTPUT}"
+
+LOG_FILTER=$(mktemp)
+LOG_SELECT=$(mktemp)
+trap 'rm -f "${LOG_FILTER}" "${LOG_SELECT}"' EXIT
+
+"${BINARY}" filter \
+    --output "${OUTPUT}" \
+    --force \
+    --ingroup "species=${SPECIES}" \
+    --outgroup all \
+    --min-frac 0.5 \
+    --max-frac 1.0 \
+    --max-outgroup-count 0 \
+    "${SOURCE}" \
+    2>"${LOG_FILTER}"
+
+cat "${LOG_FILTER}" >&2
+
+"${BINARY}" select \
+    --in-place \
+    --group "${SPECIES}:species=${SPECIES}" \
+    --group-op "${SPECIES}:any" \
+    --select "${SPECIES}" \
+    "${OUTPUT}" \
+    2>"${LOG_SELECT}"
+
+cat "${LOG_SELECT}" >&2
+
+python3 - "${SPECIES}" "${LOG_FILTER}" "${LOG_SELECT}" <<'PYEOF' >"${STATS_FILE}"
+import sys, re
+
+species, log_filter, log_select = sys.argv[1], sys.argv[2], sys.argv[3]
+
+def strip_ansi(s):
+    return re.sub(r'\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]', '', s)
+
+def parse_wall(s):
+    s = s.strip()
+    if s.endswith('ms'): return float(s[:-2]) / 1000.0
+    if s.endswith('s'):  return float(s[:-1])
+    return 0.0
+
+def parse_rss(s):
+    m = re.match(r'([\d.]+)\s*(GB|MB|KB|B)', s.strip())
+    if not m: return 0
+    return int(float(m.group(1)) * {'GB': 1<<30, 'MB': 1<<20, 'KB': 1024, 'B': 1}[m.group(2)])
+
+def is_sep(s):
+    return bool(s) and not re.search(r'[A-Za-z0-9]', s)
+
+def parse_reporter(logfile):
+    stats = {}
+    state = 'scan'
+    with open(logfile, errors='replace') as fh:
+        for raw in fh:
+            line = strip_ansi(raw.rstrip('\n'))
+            s    = line.strip()
+            if state == 'scan':
+                if re.search(r'\bstage\b.*\bwall\b', line):
+                    state = 'in_header'
+            elif state == 'in_header':
+                if is_sep(s): state = 'rows'
+            elif state == 'rows':
+                if is_sep(s): state = 'total'
+                elif s:
+                    parts = re.split(r'  +', s)
+                    if len(parts) >= 4:
+                        stats[parts[0]] = (parse_wall(parts[1]), parse_rss(parts[3]))
+            elif state == 'total':
+                if s:
+                    parts = re.split(r'  +', s)
+                    if len(parts) >= 3:
+                        stats['TOTAL'] = (parse_wall(parts[1]),
+                                          parse_rss(parts[3]) if len(parts) > 3 else 0)
+                break
+    return stats
+
+f = parse_reporter(log_filter)
+s = parse_reporter(log_select)
+
+row = [species]
+for stage, d in [('rebuild', f), ('pack', f), ('filter_total', f), ('select', s), ('select_total', s)]:
+    key = 'TOTAL' if stage.endswith('_total') else stage
+    w, r = d.get(key, ('', ''))
+    row += [f'{w:.3f}' if isinstance(w, float) else '', str(r)]
+print(','.join(row))
+PYEOF

benchmark/index_one_count.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+# Usage: index_one_count.sh SPECIMEN
+# SPECIMEN = "species--strain" (Make pattern stem)
+# Outputs:
+#   specimen_index_count/SPECIMEN/index.done  (written by obikmer)
+#   stats/indexing_count/SPECIMEN.stats       (one CSV data row, no header)
+set -euo pipefail
+
+SPECIMEN="$1"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
+
+species="${SPECIMEN%%--*}"
+strain="${SPECIMEN#*--}"
+
+READS_DIR="${SCRIPT_DIR}/simulated_data/${species}/${strain}"
+INDEX_PATH="${SCRIPT_DIR}/specimen_index_count/${SPECIMEN}"
+STATS_DIR="${SCRIPT_DIR}/stats/indexing_count"
+STATS_FILE="${STATS_DIR}/${SPECIMEN}.stats"
+
+mkdir -p "${STATS_DIR}"
+
+r1="${READS_DIR}/reads_R1.fastq.gz"
+r2="${READS_DIR}/reads_R2.fastq.gz"
+if [[ ! -f "${r1}" || ! -f "${r2}" ]]; then
+    echo "ERROR: reads not found in ${READS_DIR}" >&2
+    exit 1
+fi
+
+echo "[${SPECIMEN}] indexing (count) → ${INDEX_PATH}"
+
+STDERR_LOG=$(mktemp)
+trap 'rm -f "${STDERR_LOG}"' EXIT
+
+"${BINARY}" index \
+    --output "${INDEX_PATH}" \
+    --force \
+    --theta 0 \
+    --with-counts \
+    --label "${SPECIMEN}" \
+    --meta  "species=${species}" \
+    "${r1}" "${r2}" \
+    2>"${STDERR_LOG}"
+
+cat "${STDERR_LOG}" >&2
+
+python3 - "${species}" "${strain}" "${STDERR_LOG}" <<'PYEOF' >"${STATS_FILE}"
+import sys, re
+
+species, strain, logfile = sys.argv[1], sys.argv[2], sys.argv[3]
+
+def strip_ansi(s):
+    return re.sub(r'\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]', '', s)
+
+def parse_wall(s):
+    s = s.strip()
+    if s.endswith('ms'): return float(s[:-2]) / 1000.0
+    if s.endswith('s'):  return float(s[:-1])
+    return 0.0
+
+def parse_rss(s):
+    m = re.match(r'([\d.]+)\s*(GB|MB|KB|B)', s.strip())
+    if not m: return 0
+    return int(float(m.group(1)) * {'GB': 1<<30, 'MB': 1<<20, 'KB': 1024, 'B': 1}[m.group(2)])
+
+def is_sep(s):
+    return bool(s) and not re.search(r'[A-Za-z0-9]', s)
+
+stats = {}
+state = 'scan'
+
+with open(logfile, errors='replace') as fh:
+    for raw in fh:
+        line = strip_ansi(raw.rstrip('\n'))
+        s    = line.strip()
+        if state == 'scan':
+            if re.search(r'\bstage\b.*\bwall\b', line):
+                state = 'in_header'
+        elif state == 'in_header':
+            if is_sep(s): state = 'rows'
+        elif state == 'rows':
+            if is_sep(s): state = 'total'
+            elif s:
+                parts = re.split(r'  +', s)
+                if len(parts) >= 4:
+                    stats[parts[0]] = (parse_wall(parts[1]), parse_rss(parts[3]))
+        elif state == 'total':
+            if s:
+                parts = re.split(r'  +', s)
+                if len(parts) >= 3:
+                    stats[parts[0]] = (parse_wall(parts[1]),
+                                       parse_rss(parts[3]) if len(parts) > 3 else 0)
+            break
+
+STAGE_ORDER = ['scatter', 'dereplicate', 'count_kmer', 'index']
+row = [species, strain]
+for stage in STAGE_ORDER:
+    w, r = stats.get(stage, ('', ''))
+    row += [f'{w:.3f}' if isinstance(w, float) else '', str(r)]
+tw, tr = stats.get('TOTAL', ('', ''))
+row += [f'{tw:.3f}' if isinstance(tw, float) else '', str(tr)]
+print(','.join(row))
+PYEOF

benchmark/index_one_presence.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# Usage: index_one_presence.sh SPECIMEN
+# SPECIMEN = "species--strain" (Make pattern stem)
+# Outputs:
+#   specimen_index_presence/SPECIMEN/index.done  (written by obikmer)
+#   stats/indexing_presence/SPECIMEN.stats       (one CSV data row, no header)
+set -euo pipefail
+
+SPECIMEN="$1"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
+
+species="${SPECIMEN%%--*}"
+strain="${SPECIMEN#*--}"
+
+READS_DIR="${SCRIPT_DIR}/simulated_data/${species}/${strain}"
+INDEX_PATH="${SCRIPT_DIR}/specimen_index_presence/${SPECIMEN}"
+STATS_DIR="${SCRIPT_DIR}/stats/indexing_presence"
+STATS_FILE="${STATS_DIR}/${SPECIMEN}.stats"
+
+mkdir -p "${STATS_DIR}"
+
+r1="${READS_DIR}/reads_R1.fastq.gz"
+r2="${READS_DIR}/reads_R2.fastq.gz"
+if [[ ! -f "${r1}" || ! -f "${r2}" ]]; then
+    echo "ERROR: reads not found in ${READS_DIR}" >&2
+    exit 1
+fi
+
+echo "[${SPECIMEN}] indexing (presence) → ${INDEX_PATH}"
+
+STDERR_LOG=$(mktemp)
+trap 'rm -f "${STDERR_LOG}"' EXIT
+
+"${BINARY}" index \
+    --output "${INDEX_PATH}" \
+    --force \
+    --theta 0 \
+    --label "${SPECIMEN}" \
+    --meta  "species=${species}" \
+    "${r1}" "${r2}" \
+    2>"${STDERR_LOG}"
+
+cat "${STDERR_LOG}" >&2
+
+python3 - "${species}" "${strain}" "${STDERR_LOG}" <<'PYEOF' >"${STATS_FILE}"
+import sys, re
+
+species, strain, logfile = sys.argv[1], sys.argv[2], sys.argv[3]
+
+def strip_ansi(s):
+    return re.sub(r'\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]', '', s)
+
+def parse_wall(s):
+    s = s.strip()
+    if s.endswith('ms'): return float(s[:-2]) / 1000.0
+    if s.endswith('s'):  return float(s[:-1])
+    return 0.0
+
+def parse_rss(s):
+    m = re.match(r'([\d.]+)\s*(GB|MB|KB|B)', s.strip())
+    if not m: return 0
+    return int(float(m.group(1)) * {'GB': 1<<30, 'MB': 1<<20, 'KB': 1024, 'B': 1}[m.group(2)])
+
+def is_sep(s):
+    return bool(s) and not re.search(r'[A-Za-z0-9]', s)
+
+stats = {}
+state = 'scan'
+
+with open(logfile, errors='replace') as fh:
+    for raw in fh:
+        line = strip_ansi(raw.rstrip('\n'))
+        s    = line.strip()
+        if state == 'scan':
+            if re.search(r'\bstage\b.*\bwall\b', line):
+                state = 'in_header'
+        elif state == 'in_header':
+            if is_sep(s): state = 'rows'
+        elif state == 'rows':
+            if is_sep(s): state = 'total'
+            elif s:
+                parts = re.split(r'  +', s)
+                if len(parts) >= 4:
+                    stats[parts[0]] = (parse_wall(parts[1]), parse_rss(parts[3]))
+        elif state == 'total':
+            if s:
+                parts = re.split(r'  +', s)
+                if len(parts) >= 3:
+                    stats[parts[0]] = (parse_wall(parts[1]),
+                                       parse_rss(parts[3]) if len(parts) > 3 else 0)
+            break
+
+STAGE_ORDER = ['scatter', 'dereplicate', 'count_kmer', 'index']
+row = [species, strain]
+for stage in STAGE_ORDER:
+    w, r = stats.get(stage, ('', ''))
+    row += [f'{w:.3f}' if isinstance(w, float) else '', str(r)]
+tw, tr = stats.get('TOTAL', ('', ''))
+row += [f'{tw:.3f}' if isinstance(tw, float) else '', str(tr)]
+print(','.join(row))
+PYEOF

benchmark/make_deps.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Generate deps.mk — pure dependency declarations for the benchmark pipeline.
+
+Like C .d files: only target: prerequisites lines, no recipes.
+Recipes stay in the Makefile as generic rules.
+"""
+import gzip
+import re
+import sys
+from pathlib import Path
+
+STOP_WORDS    = {'complete', 'chromosome', 'whole', 'sequence', 'genome',
+                 'endosymbiont', 'of'}
+STOP_PREFIXES = ('scaffold', 'contig', 'plasmid')
+
+
+def is_stop(tok):
+    t = tok.lower()
+    return t in STOP_WORDS or any(t.startswith(p) for p in STOP_PREFIXES)
+
+
+def sanitize(s):
+    return re.sub(r'[^A-Za-z0-9._-]', '_', s).strip('_')
+
+
+def collect_tokens(text):
+    parts = []
+    for tok in text.split():
+        tok = tok.rstrip(',.')
+        if is_stop(tok):
+            break
+        parts.append(sanitize(tok))
+    return '_'.join(filter(None, parts))
+
+
+def parse_organism(defn, gcf_id):
+    words   = defn.split()
+    species = sanitize(words[0] + '_' + words[1])
+
+    m = re.search(r'\bstr\.\s+(\S+)(?:\s+substr\.\s+(\S+))?', defn)
+    if m:
+        strain = sanitize(m.group(1))
+        if m.group(2):
+            strain += '_' + sanitize(m.group(2))
+        return species, strain
+
+    m = re.search(r'\bstrain\b\s+(.*)', defn)
+    if m:
+        strain = collect_tokens(m.group(1))
+        if strain:
+            return species, strain
+
+    remainder = re.sub(r'^\S+ \S+\s*', '', defn)
+    remainder = re.sub(r'^subsp\.\s+\S+\s*', '', remainder)
+    remainder = re.sub(r'^serovar\s+\S+\s*', '', remainder)
+    strain    = collect_tokens(remainder)
+    return species, strain if strain else gcf_id
+
+
+def first_definition(path):
+    with gzip.open(path, 'rt') as fh:
+        for line in fh:
+            if line.startswith('>'):
+                m = re.search(r'"definition":"([^"]*)"', line)
+                return m.group(1) if m else line[1:].split()[0]
+    return Path(path).stem
+
+
+def main():
+    entries = []   # (specimen, species, sim_dir, genome_path)
+    species_seen = []
+
+    for path in sorted(sys.argv[1:]):
+        gcf_id  = Path(path).name.replace('_genomic.fna.gz', '')
+        defn    = first_definition(path)
+        sp, st  = parse_organism(defn, gcf_id)
+        specimen = f'{sp}--{st}'
+        sim_dir  = f'simulated_data/{sp}/{st}'
+        entries.append((specimen, sp, sim_dir, path))
+        if sp not in species_seen:
+            species_seen.append(sp)
+
+    specimens = [e[0] for e in entries]
+    print('SPECIMENS :=', ' '.join(specimens))
+    print('SPECIES   :=', ' '.join(species_seen))
+
+    for specimen, species, sim_dir, genome in entries:
+        reads = f'{sim_dir}/reads_R1.fastq.gz'
+        p_done  = f'specimen_index_presence/{specimen}/index.done'
+        p_stats = f'stats/indexing_presence/{specimen}.stats'
+        c_done  = f'specimen_index_count/{specimen}/index.done'
+        c_stats = f'stats/indexing_count/{specimen}.stats'
+        ref     = f'reference_index/{specimen}.npz'
+        vp      = f'stats/verify_presence/{specimen}.stats'
+        vc      = f'stats/verify_count/{specimen}.stats'
+
+        print()
+        print(f'# {specimen}')
+        print(f'{reads}: {genome}')
+        print(f'{ref}: {reads}')
+        print(f'{p_done} {p_stats}: {reads}')
+        print(f'{c_done} {c_stats}: {reads}')
+        print(f'{vp}: {ref} {p_done}')
+        print(f'{vc}: {ref} {c_done}')
+
+    print()
+    for sp in species_seen:
+        sp_done  = f'specific_index_presence/{sp}/index.done'
+        sp_stats = f'stats/specific_kmer_presence/{sp}.stats'
+        sc_done  = f'specific_index_count/{sp}/index.done'
+        sc_stats = f'stats/specific_kmer_count/{sp}.stats'
+        print(f'# {sp}')
+        print(f'{sp_done} {sp_stats}: global_index_presence/index.done')
+        print(f'{sc_done} {sc_stats}: global_index_count/index.done')
+
+
+if __name__ == '__main__':
+    main()

benchmark/merge_count.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
+IDX_DIR="${SCRIPT_DIR}/specimen_index_count"
+OUTPUT="${SCRIPT_DIR}/global_index_count"
+STATS_DIR="${SCRIPT_DIR}/stats/merge_count"
+
+mkdir -p "${STATS_DIR}"
+
+run_n=$(printf '%03d' "$(find "${STATS_DIR}" -maxdepth 1 -name 'run_*.csv' | wc -l | tr -d ' ')")
+CSV="${STATS_DIR}/run_${run_n}.csv"
+
+printf 'run,n_sources,bootstrap_wall_s,bootstrap_rss_b,spectrums_wall_s,spectrums_rss_b,merge_partitions_wall_s,merge_partitions_rss_b,pack_wall_s,pack_rss_b,total_wall_s,total_rss_b\n' >"${CSV}"
+
+parse_reporter() {
+    local run="$1" n_sources="$2" logfile="$3"
+    python3 - "$run" "$n_sources" "$logfile" <<'PYEOF'
+import sys, re
+
+run, n_sources, logfile = sys.argv[1], sys.argv[2], sys.argv[3]
+
+def strip_ansi(s):
+    return re.sub(r'\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]', '', s)
+
+def parse_wall(s):
+    s = s.strip()
+    if s.endswith('ms'): return float(s[:-2]) / 1000.0
+    if s.endswith('s'):  return float(s[:-1])
+    return 0.0
+
+def parse_rss(s):
+    m = re.match(r'([\d.]+)\s*(GB|MB|KB|B)', s.strip())
+    if not m: return 0
+    return int(float(m.group(1)) * {'GB': 1<<30, 'MB': 1<<20, 'KB': 1024, 'B': 1}[m.group(2)])
+
+def is_sep(s):
+    return bool(s) and not re.search(r'[A-Za-z0-9]', s)
+
+stats = {}
+state = 'scan'
+
+with open(logfile, errors='replace') as fh:
+    for raw in fh:
+        line = strip_ansi(raw.rstrip('\n'))
+        s    = line.strip()
+
+        if state == 'scan':
+            if re.search(r'\bstage\b.*\bwall\b', line):
+                state = 'in_header'
+        elif state == 'in_header':
+            if is_sep(s):
+                state = 'rows'
+        elif state == 'rows':
+            if is_sep(s):
+                state = 'total'
+            elif s:
+                parts = re.split(r'  +', s)
+                if len(parts) >= 4:
+                    stats[parts[0]] = (parse_wall(parts[1]), parse_rss(parts[3]))
+        elif state == 'total':
+            if s:
+                parts = re.split(r'  +', s)
+                if len(parts) >= 3:
+                    stats[parts[0]] = (parse_wall(parts[1]),
+                                       parse_rss(parts[3]) if len(parts) > 3 else 0)
+            break
+
+STAGE_ORDER = ['bootstrap', 'spectrums', 'merge_partitions', 'pack']
+row = [run, n_sources]
+for stage in STAGE_ORDER:
+    w, r = stats.get(stage, ('', ''))
+    row += [f'{w:.3f}' if isinstance(w, float) else '', str(r)]
+tw, tr = stats.get('TOTAL', ('', ''))
+row += [f'{tw:.3f}' if isinstance(tw, float) else '', str(tr)]
+print(','.join(row))
+PYEOF
+}
+
+mapfile -t sources < <(find "${IDX_DIR}" -mindepth 1 -maxdepth 1 -type d | sort)
+
+if [[ ${#sources[@]} -eq 0 ]]; then
+    echo "ERROR: no indexes found in ${IDX_DIR}" >&2
+    exit 1
+fi
+
+echo "Merging ${#sources[@]} count indexes → ${OUTPUT}"
+printf '  %s\n' "${sources[@]}"
+
+STDERR_LOG=$(mktemp)
+trap 'rm -f "${STDERR_LOG}"' EXIT
+
+"${BINARY}" merge \
+    --output  "${OUTPUT}" \
+    --force \
+    "${sources[@]}" \
+    2>"${STDERR_LOG}"
+
+cat "${STDERR_LOG}" >&2
+parse_reporter "${run_n}" "${#sources[@]}" "${STDERR_LOG}" >>"${CSV}"
+
+echo "Done. Run ${run_n} → ${CSV}"

benchmark/merge_presence.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
+IDX_DIR="${SCRIPT_DIR}/specimen_index_presence"
+OUTPUT="${SCRIPT_DIR}/global_index_presence"
+STATS_DIR="${SCRIPT_DIR}/stats/merge_presence"
+
+mkdir -p "${STATS_DIR}"
+
+run_n=$(printf '%03d' "$(find "${STATS_DIR}" -maxdepth 1 -name 'run_*.csv' | wc -l | tr -d ' ')")
+CSV="${STATS_DIR}/run_${run_n}.csv"
+
+printf 'run,n_sources,bootstrap_wall_s,bootstrap_rss_b,spectrums_wall_s,spectrums_rss_b,merge_partitions_wall_s,merge_partitions_rss_b,pack_wall_s,pack_rss_b,total_wall_s,total_rss_b\n' >"${CSV}"
+
+parse_reporter() {
+    local run="$1" n_sources="$2" logfile="$3"
+    python3 - "$run" "$n_sources" "$logfile" <<'PYEOF'
+import sys, re
+
+run, n_sources, logfile = sys.argv[1], sys.argv[2], sys.argv[3]
+
+def strip_ansi(s):
+    return re.sub(r'\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]', '', s)
+
+def parse_wall(s):
+    s = s.strip()
+    if s.endswith('ms'): return float(s[:-2]) / 1000.0
+    if s.endswith('s'):  return float(s[:-1])
+    return 0.0
+
+def parse_rss(s):
+    m = re.match(r'([\d.]+)\s*(GB|MB|KB|B)', s.strip())
+    if not m: return 0
+    return int(float(m.group(1)) * {'GB': 1<<30, 'MB': 1<<20, 'KB': 1024, 'B': 1}[m.group(2)])
+
+def is_sep(s):
+    return bool(s) and not re.search(r'[A-Za-z0-9]', s)
+
+stats = {}
+state = 'scan'
+
+with open(logfile, errors='replace') as fh:
+    for raw in fh:
+        line = strip_ansi(raw.rstrip('\n'))
+        s    = line.strip()
+
+        if state == 'scan':
+            if re.search(r'\bstage\b.*\bwall\b', line):
+                state = 'in_header'
+        elif state == 'in_header':
+            if is_sep(s):
+                state = 'rows'
+        elif state == 'rows':
+            if is_sep(s):
+                state = 'total'
+            elif s:
+                parts = re.split(r'  +', s)
+                if len(parts) >= 4:
+                    stats[parts[0]] = (parse_wall(parts[1]), parse_rss(parts[3]))
+        elif state == 'total':
+            if s:
+                parts = re.split(r'  +', s)
+                if len(parts) >= 3:
+                    stats[parts[0]] = (parse_wall(parts[1]),
+                                       parse_rss(parts[3]) if len(parts) > 3 else 0)
+            break
+
+STAGE_ORDER = ['bootstrap', 'spectrums', 'merge_partitions', 'pack']
+row = [run, n_sources]
+for stage in STAGE_ORDER:
+    w, r = stats.get(stage, ('', ''))
+    row += [f'{w:.3f}' if isinstance(w, float) else '', str(r)]
+tw, tr = stats.get('TOTAL', ('', ''))
+row += [f'{tw:.3f}' if isinstance(tw, float) else '', str(tr)]
+print(','.join(row))
+PYEOF
+}
+
+mapfile -t sources < <(find "${IDX_DIR}" -mindepth 1 -maxdepth 1 -type d | sort)
+
+if [[ ${#sources[@]} -eq 0 ]]; then
+    echo "ERROR: no indexes found in ${IDX_DIR}" >&2
+    exit 1
+fi
+
+echo "Merging ${#sources[@]} presence indexes → ${OUTPUT}"
+printf '  %s\n' "${sources[@]}"
+
+STDERR_LOG=$(mktemp)
+trap 'rm -f "${STDERR_LOG}"' EXIT
+
+"${BINARY}" merge \
+    --output          "${OUTPUT}" \
+    --force \
+    --force-presence \
+    "${sources[@]}" \
+    2>"${STDERR_LOG}"
+
+cat "${STDERR_LOG}" >&2
+parse_reporter "${run_n}" "${#sources[@]}" "${STDERR_LOG}" >>"${CSV}"
+
+echo "Done. Run ${run_n} → ${CSV}"

benchmark/simulate.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Simulate all genomes. Delegates to simulate_one.sh per genome.
+# Prefer running via `gmake simulate` which handles individual dependencies.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+for genome_file in "${SCRIPT_DIR}"/genomes/*.fna.gz; do
+    out_dir=$("${SCRIPT_DIR}/../.venv/bin/python3" "${SCRIPT_DIR}/make_deps.py" \
+        --dir-for "${genome_file}")
+    bash "${SCRIPT_DIR}/simulate_one.sh" "${genome_file}" "${out_dir}"
+done

benchmark/simulate_one.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Usage: simulate_one.sh genome.fna.gz output_dir
+# Simulates paired-end HiSeq reads for a single genome.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ISS="${SCRIPT_DIR}/../.venv/bin/iss"
+COVERAGE=15
+READ_LENGTH=150
+CPUS="${CPUS:-$(sysctl -n hw.logicalcpu 2>/dev/null || nproc 2>/dev/null || echo 2)}"
+
+genome_file="$1"
+out_dir="$2"
+
+mkdir -p "${out_dir}"
+
+tmp_fasta=$(mktemp "${TMPDIR:-/tmp}/obikmer_XXXXXX.fna")
+trap 'rm -f "${tmp_fasta}"' EXIT
+
+gzip -dc "${genome_file}" > "${tmp_fasta}"
+
+genome_size=$(grep -v "^>" "${tmp_fasta}" | tr -d '[:space:]' | wc -c | tr -d ' ')
+n_reads=$(python3 -c "import math; print(math.ceil(${COVERAGE} * ${genome_size} / (2 * ${READ_LENGTH})))")
+
+echo "[${out_dir}]  genome=${genome_size} bp  →  ${n_reads} read pairs  (${COVERAGE}x HiSeq)"
+
+"${ISS}" generate \
+    --genomes   "${tmp_fasta}" \
+    --model     HiSeq \
+    --n_reads   "${n_reads}" \
+    --cpus      "${CPUS}" \
+    --compress \
+    --output    "${out_dir}/reads"

benchmark/test_dist.csv

@@ -0,0 +1,21 @@
+genome,Candidozyma_auris--GCF_003013715.1_ASM301371v2,Acidobacterium_capsulatum--ATCC_51196,Bacillus_subtilis--168,Escherichia_coli--CFT073,Escherichia_coli--EDL933,Escherichia_coli--K-12_MG1655,Escherichia_coli--K-12_W3110,Klebsiella_pneumoniae--ATCC_13883,Klebsiella_pneumoniae--HS11286,Klebsiella_pneumoniae--MGH_78578,Opitutus_terrae--PB90-1,Proteus_mirabilis--HI4320,Saccharolobus_islandicus--M.16.4,Salmonella_enterica--AKU_12601,Salmonella_enterica--CT18,Salmonella_enterica--LT2,Salmonella_enterica--P125109,Shouchella_clausii--KSM-K16,Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1,Yersinia_ruckeri--YRB
+Candidozyma_auris--GCF_003013715.1_ASM301371v2,0.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000
+Acidobacterium_capsulatum--ATCC_51196,1.000000,0.000000,0.999981,0.999990,0.999989,0.999987,0.999987,0.999990,0.999988,0.999988,0.999994,0.999989,1.000000,0.999988,0.999987,0.999987,0.999988,0.999989,0.999991,0.999987
+Bacillus_subtilis--168,1.000000,0.999981,0.000000,0.999990,0.999989,0.999989,0.999989,0.999989,0.999988,0.999986,0.999995,0.999985,0.999999,0.999988,0.999987,0.999989,0.999988,0.999778,0.999993,0.999987
+Escherichia_coli--CFT073,1.000000,0.999990,0.999990,0.000000,0.825741,0.807495,0.807218,0.991156,0.996855,0.997849,0.999996,0.999633,1.000000,0.993885,0.996736,0.994148,0.993821,0.999991,0.999984,0.999291
+Escherichia_coli--EDL933,1.000000,0.999989,0.999989,0.825741,0.000000,0.735107,0.734775,0.996126,0.998058,0.997908,0.999997,0.999640,1.000000,0.993993,0.997126,0.994390,0.994059,0.999991,0.999986,0.999292
+Escherichia_coli--K-12_MG1655,1.000000,0.999987,0.999989,0.807495,0.735107,0.000000,0.382567,0.996190,0.997747,0.997455,0.999996,0.999604,1.000000,0.993444,0.996645,0.993773,0.993431,0.999989,0.999984,0.999174
+Escherichia_coli--K-12_W3110,1.000000,0.999987,0.999989,0.807218,0.734775,0.382567,0.000000,0.996220,0.997761,0.997467,0.999995,0.999604,1.000000,0.993445,0.996669,0.993769,0.993443,0.999990,0.999985,0.999165
+Klebsiella_pneumoniae--ATCC_13883,1.000000,0.999990,0.999989,0.991156,0.996126,0.996190,0.996220,0.000000,0.845220,0.840545,0.999997,0.999648,1.000000,0.996177,0.998128,0.996268,0.996052,0.999990,0.999987,0.999325
+Klebsiella_pneumoniae--HS11286,1.000000,0.999988,0.999988,0.996855,0.998058,0.997747,0.997761,0.845220,0.000000,0.906475,0.999996,0.999683,1.000000,0.997724,0.995697,0.997776,0.997769,0.999989,0.999979,0.999463
+Klebsiella_pneumoniae--MGH_78578,1.000000,0.999988,0.999986,0.997849,0.997908,0.997455,0.997467,0.840545,0.906475,0.000000,0.999996,0.999704,1.000000,0.997928,0.995054,0.997844,0.997868,0.999990,0.999980,0.999479
+Opitutus_terrae--PB90-1,1.000000,0.999994,0.999995,0.999996,0.999997,0.999996,0.999995,0.999997,0.999996,0.999996,0.000000,0.999997,0.999998,0.999996,0.999996,0.999996,0.999995,0.999997,0.999993,0.999996
+Proteus_mirabilis--HI4320,1.000000,0.999989,0.999985,0.999633,0.999640,0.999604,0.999604,0.999648,0.999683,0.999704,0.999997,0.000000,1.000000,0.999604,0.999699,0.999622,0.999613,0.999987,0.999983,0.999505
+Saccharolobus_islandicus--M.16.4,1.000000,1.000000,0.999999,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,0.999998,1.000000,0.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000,1.000000
+Salmonella_enterica--AKU_12601,1.000000,0.999988,0.999988,0.993885,0.993993,0.993444,0.993445,0.996177,0.997724,0.997928,0.999996,0.999604,1.000000,0.000000,0.869238,0.682277,0.663383,0.999990,0.999985,0.999260
+Salmonella_enterica--CT18,1.000000,0.999987,0.999987,0.996736,0.997126,0.996645,0.996669,0.998128,0.995697,0.995054,0.999996,0.999699,1.000000,0.869238,0.000000,0.890872,0.886148,0.999988,0.999976,0.999524
+Salmonella_enterica--LT2,1.000000,0.999987,0.999989,0.994148,0.994390,0.993773,0.993769,0.996268,0.997776,0.997844,0.999996,0.999622,1.000000,0.682277,0.890872,0.000000,0.622606,0.999989,0.999985,0.999296
+Salmonella_enterica--P125109,1.000000,0.999988,0.999988,0.993821,0.994059,0.993431,0.993443,0.996052,0.997769,0.997868,0.999995,0.999613,1.000000,0.663383,0.886148,0.622606,0.000000,0.999988,0.999983,0.999270
+Shouchella_clausii--KSM-K16,1.000000,0.999989,0.999778,0.999991,0.999991,0.999989,0.999990,0.999990,0.999989,0.999990,0.999997,0.999987,1.000000,0.999990,0.999988,0.999989,0.999988,0.000000,0.999991,0.999988
+Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1,1.000000,0.999991,0.999993,0.999984,0.999986,0.999984,0.999985,0.999987,0.999979,0.999980,0.999993,0.999983,1.000000,0.999985,0.999976,0.999985,0.999983,0.999991,0.000000,0.999983
+Yersinia_ruckeri--YRB,1.000000,0.999987,0.999987,0.999291,0.999292,0.999174,0.999165,0.999325,0.999463,0.999479,0.999996,0.999505,1.000000,0.999260,0.999524,0.999296,0.999270,0.999988,0.999983,0.000000

benchmark/test_nj.nwk

@@ -0,0 +1 @@
+(((((((((((Candidozyma_auris--GCF_003013715.1_ASM301371v2:0.5000001881725941,Saccharolobus_islandicus--M.16.4:0.4999993211600824):0.0000023411501775538747,Opitutus_terrae--PB90-1:0.499997075187947):0.0000029791191795691675,(Acidobacterium_capsulatum--ATCC_51196:0.49999227771334689,(Bacillus_subtilis--168:0.49988797935621456,Shouchella_clausii--KSM-K16:0.49988984146059159):0.0001037210285571577):0.0000023959836053522034):0.0000034093646568700288,Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1:0.4999920159222422):0.000199555100890203,Proteus_mirabilis--HI4320:0.49979129185300427):0.00010103619067070024,Yersinia_ruckeri--YRB:0.4996806650749249):0.0013719139155004,(Klebsiella_pneumoniae--HS11286:0.43798845051648258,(Klebsiella_pneumoniae--ATCC_13883:0.41780293826821265,Klebsiella_pneumoniae--MGH_78578:0.42274184870836559):0.017586732339732737):0.0604124197073832):0.0006482538063555254,(Salmonella_enterica--CT18:0.43952894448143017,(Salmonella_enterica--AKU_12601:0.3357977326267918,(Salmonella_enterica--LT2:0.31203395843666389,Salmonella_enterica--P125109:0.31057217324861216):0.025729515856701136):0.10292985918524672):0.05825411485542886):0.08937928015651564,Escherichia_coli--CFT073:0.40806501650701029):0.0410131211869626,Escherichia_coli--EDL933:0.3681464750911808):0.1755112579711463,Escherichia_coli--K-12_MG1655:0.19129818036662728,Escherichia_coli--K-12_W3110:0.19126872019906239);

benchmark/test_shared.csv

@@ -0,0 +1,21 @@
+genome,Candidozyma_auris--GCF_003013715.1_ASM301371v2,Acidobacterium_capsulatum--ATCC_51196,Bacillus_subtilis--168,Escherichia_coli--CFT073,Escherichia_coli--EDL933,Escherichia_coli--K-12_MG1655,Escherichia_coli--K-12_W3110,Klebsiella_pneumoniae--ATCC_13883,Klebsiella_pneumoniae--HS11286,Klebsiella_pneumoniae--MGH_78578,Opitutus_terrae--PB90-1,Proteus_mirabilis--HI4320,Saccharolobus_islandicus--M.16.4,Salmonella_enterica--AKU_12601,Salmonella_enterica--CT18,Salmonella_enterica--LT2,Salmonella_enterica--P125109,Shouchella_clausii--KSM-K16,Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1,Yersinia_ruckeri--YRB
+Candidozyma_auris--GCF_003013715.1_ASM301371v2,0,0,0,0,0,0,0,0,0,0,0,0,8,0,1,0,0,0,0,3
+Acidobacterium_capsulatum--ATCC_51196,0,0,203,119,128,141,140,116,109,111,78,112,0,136,109,147,134,117,55,129
+Bacillus_subtilis--168,0,203,0,124,132,128,123,133,109,130,66,158,6,131,112,124,135,2393,46,124
+Escherichia_coli--CFT073,0,119,124,0,1966777,1998059,1999094,117743,32029,22312,63,4225,0,74946,31918,73311,76585,113,128,7854
+Escherichia_coli--EDL933,0,128,132,1966777,0,2627885,2628700,52488,20134,22064,48,4202,0,74655,28602,71244,74665,112,108,7963
+Escherichia_coli--K-12_MG1655,0,141,128,1998059,2627885,0,4452541,48302,21382,24602,47,4277,0,75729,30449,73622,76778,119,111,8566
+Escherichia_coli--K-12_W3110,0,140,123,1999094,2628700,4452541,0,47894,21226,24470,68,4278,0,75658,30207,73614,76583,112,108,8660
+Klebsiella_pneumoniae--ATCC_13883,0,116,133,117743,52488,48302,47894,0,1416091,1477759,42,4172,0,48296,18988,48144,50416,120,106,7712
+Klebsiella_pneumoniae--HS11286,0,109,109,32029,20134,21382,21226,1416091,0,644063,42,2738,0,21498,29758,21606,21376,99,102,4417
+Klebsiella_pneumoniae--MGH_78578,0,111,130,22312,22064,24602,24470,1477759,644063,0,42,2614,0,19948,35067,21330,20813,97,102,4374
+Opitutus_terrae--PB90-1,0,78,66,63,48,47,68,42,42,42,0,43,18,57,42,53,66,39,58,43
+Proteus_mirabilis--HI4320,0,112,158,4225,4202,4277,4278,4172,2738,2614,43,0,0,4254,2481,4166,4215,131,103,4704
+Saccharolobus_islandicus--M.16.4,8,0,6,0,0,0,0,0,0,0,18,0,0,0,0,0,0,0,0,0
+Salmonella_enterica--AKU_12601,0,136,131,74946,74655,75729,75658,48296,21498,19948,57,4254,0,0,1047731,2857146,2951421,117,108,7643
+Salmonella_enterica--CT18,1,109,112,31918,28602,30449,30207,18988,29758,35067,42,2481,0,1047731,0,917948,940297,106,106,3716
+Salmonella_enterica--LT2,0,147,124,73311,71244,73622,73614,48144,21606,21330,53,4166,0,2857146,917948,0,3284800,122,108,7460
+Salmonella_enterica--P125109,0,134,135,76585,74665,76778,76583,50416,21376,20813,66,4215,0,2951421,940297,3284800,0,134,124,7645
+Shouchella_clausii--KSM-K16,0,117,2393,113,112,119,112,120,99,97,39,131,0,117,106,122,134,0,58,124
+Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1,0,55,46,128,108,111,108,106,102,102,58,103,0,108,106,108,124,58,0,96
+Yersinia_ruckeri--YRB,3,129,124,7854,7963,8566,8660,7712,4417,4374,43,4704,0,7643,3716,7460,7645,124,96,0

benchmark/verify_count.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""Compare an obikmer count index against a reference kmer set (presence + counts).
+
+Loads the reference .npz (sorted uint64 kmers + uint32 counts from build_reference.py),
+streams `obikmer dump` from a --with-counts index, then reports:
+  - false negatives : kmers in reference absent from the index
+  - false positives : kmers in the index absent from the reference
+  - count mismatches: kmers present in both but with differing counts
+
+Output to stdout: one CSV row
+  species,strain,ref_kmers,idx_kmers,false_neg,false_pos,count_mismatch,
+  fn_pct,fp_pct,cm_pct
+"""
+import argparse
+import subprocess
+import sys
+
+import numpy as np
+
+
+# ── encoding ──────────────────────────────────────────────────────────────────
+
+_ENCODE = {'A': 0, 'C': 1, 'G': 2, 'T': 3,
+           'a': 0, 'c': 1, 'g': 2, 't': 3}
+
+_DECODE = ['A', 'C', 'G', 'T']
+
+
+def encode_kmer(s: str) -> int:
+    kmer = 0
+    for c in s:
+        kmer = (kmer << 2) | _ENCODE[c]
+    return kmer
+
+
+def decode_kmer(val: int, k: int) -> str:
+    bases = []
+    for _ in range(k):
+        bases.append(_DECODE[val & 3])
+        val >>= 2
+    return ''.join(reversed(bases))
+
+
+# ── dump parsing ──────────────────────────────────────────────────────────────
+
+def load_index(obikmer_bin: str, index_dir: str) -> tuple[np.ndarray, np.ndarray]:
+    """Stream `obikmer dump` and return (kmers_sorted_uint64, counts_uint32)."""
+    cmd = [obikmer_bin, 'dump', index_dir]
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+                            text=True)
+    kmers, counts = [], []
+    header = True
+    for line in proc.stdout:
+        if header:
+            header = False
+            continue
+        parts = line.rstrip('\n').split(',')
+        kmers.append(encode_kmer(parts[0]))
+        counts.append(int(parts[1]))
+    proc.wait()
+    if proc.returncode != 0:
+        print(f'ERROR: obikmer dump exited {proc.returncode}', file=sys.stderr)
+        sys.exit(1)
+    order = np.argsort(np.array(kmers, dtype=np.uint64), kind='stable')
+    return (np.array(kmers, dtype=np.uint64)[order],
+            np.array(counts, dtype=np.uint32)[order])
+
+
+# ── comparison ────────────────────────────────────────────────────────────────
+
+def compare(ref_kmers: np.ndarray, ref_counts: np.ndarray,
+            idx_kmers: np.ndarray, idx_counts: np.ndarray,
+            ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    """Return (false_neg, false_pos, cm_ref_kmers, cm_ref_counts, cm_idx_counts).
+
+    All arrays sorted; cm_* cover kmers present in both arrays but with
+    differing counts.
+    """
+    false_neg = np.setdiff1d(ref_kmers, idx_kmers, assume_unique=True)
+    false_pos = np.setdiff1d(idx_kmers, ref_kmers, assume_unique=True)
+
+    # Count mismatches among shared kmers.
+    # Both arrays are sorted so we can use searchsorted.
+    pos_in_idx = np.searchsorted(idx_kmers, ref_kmers)
+    pos_in_idx = np.clip(pos_in_idx, 0, len(idx_kmers) - 1)
+    shared_mask = idx_kmers[pos_in_idx] == ref_kmers
+
+    shared_ref_counts = ref_counts[shared_mask]
+    shared_idx_counts = idx_counts[pos_in_idx[shared_mask]]
+    mismatch_mask     = shared_ref_counts != shared_idx_counts
+
+    cm_kmers      = ref_kmers[shared_mask][mismatch_mask]
+    cm_ref_counts = shared_ref_counts[mismatch_mask]
+    cm_idx_counts = shared_idx_counts[mismatch_mask]
+
+    return false_neg, false_pos, cm_kmers, cm_ref_counts, cm_idx_counts
+
+
+# ── main ─────────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument('reference',  metavar='REF_NPZ',   nargs='?',
+                    help='Reference .npz file')
+    ap.add_argument('index',      metavar='INDEX_DIR', nargs='?',
+                    help='obikmer index directory (built with --with-counts)')
+    ap.add_argument('--obikmer',  default='obikmer',
+                    help='Path to obikmer binary')
+    ap.add_argument('--species',  default='')
+    ap.add_argument('--strain',   default='')
+    ap.add_argument('--header',   action='store_true',
+                    help='Print CSV header and exit')
+    ap.add_argument('--save-fp',  metavar='FILE',
+                    help='Save false-positive kmer strings to FILE')
+    ap.add_argument('--save-fn',  metavar='FILE',
+                    help='Save false-negative kmer strings to FILE')
+    ap.add_argument('--save-cm',  metavar='FILE',
+                    help='Save count-mismatch rows (kmer,ref_count,idx_count) to FILE')
+    args = ap.parse_args()
+
+    if args.header:
+        print('species,strain,ref_kmers,idx_kmers,'
+              'false_neg,false_pos,count_mismatch,'
+              'fn_pct,fp_pct,cm_pct')
+        return
+
+    # Detect k
+    cmd1 = [args.obikmer, 'dump', '--head', '1', args.index]
+    out1 = subprocess.check_output(cmd1, stderr=subprocess.DEVNULL, text=True)
+    k = len(out1.splitlines()[1].split(',')[0])
+
+    # Load reference
+    print(f'Loading reference: {args.reference}', file=sys.stderr)
+    npz = np.load(args.reference)
+    ref_kmers  = npz['kmers']    # sorted uint64
+    ref_counts = npz['counts']   # uint32
+
+    # Load index
+    print(f'Streaming dump (k={k}): {args.index}', file=sys.stderr)
+    idx_kmers, idx_counts = load_index(args.obikmer, args.index)
+
+    print(f'k={k}  ref={len(ref_kmers):,}  idx={len(idx_kmers):,}', file=sys.stderr)
+
+    false_neg, false_pos, cm_kmers, cm_ref, cm_idx = compare(
+        ref_kmers, ref_counts, idx_kmers, idx_counts)
+
+    n_shared  = len(ref_kmers) - len(false_neg)
+    fn_pct    = 100.0 * len(false_neg) / len(ref_kmers) if len(ref_kmers) else 0.0
+    fp_pct    = 100.0 * len(false_pos) / len(idx_kmers) if len(idx_kmers) else 0.0
+    cm_pct    = 100.0 * len(cm_kmers)  / n_shared       if n_shared        else 0.0
+
+    print(f'false negatives : {len(false_neg):,}  ({fn_pct:.4f}%)', file=sys.stderr)
+    print(f'false positives : {len(false_pos):,}  ({fp_pct:.4f}%)', file=sys.stderr)
+    print(f'count mismatches: {len(cm_kmers):,}  ({cm_pct:.4f}% of shared)',
+          file=sys.stderr)
+
+    if args.save_fn and len(false_neg):
+        with open(args.save_fn, 'w') as fh:
+            for v in false_neg:
+                fh.write(decode_kmer(int(v), k) + '\n')
+
+    if args.save_fp and len(false_pos):
+        with open(args.save_fp, 'w') as fh:
+            for v in false_pos:
+                fh.write(decode_kmer(int(v), k) + '\n')
+
+    if args.save_cm and len(cm_kmers):
+        with open(args.save_cm, 'w') as fh:
+            fh.write('kmer,ref_count,idx_count\n')
+            for v, rc, ic in zip(cm_kmers, cm_ref, cm_idx):
+                fh.write(f'{decode_kmer(int(v), k)},{rc},{ic}\n')
+
+    print(f'{args.species},{args.strain},'
+          f'{len(ref_kmers)},{len(idx_kmers)},'
+          f'{len(false_neg)},{len(false_pos)},{len(cm_kmers)},'
+          f'{fn_pct:.4f},{fp_pct:.4f},{cm_pct:.4f}')
+
+
+if __name__ == '__main__':
+    main()

benchmark/verify_merge_count.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""Verify the merged count index against all per-specimen reference sets.
+
+Streams `obikmer dump` once on the merged index, accumulates per-specimen
+kmer+count pairs from each column, then compares each against its reference .npz.
+
+Output to stdout: one CSV row per specimen (same columns as verify_count.py)
+  species,strain,ref_kmers,idx_kmers,false_neg,false_pos,count_mismatch,
+  fn_pct,fp_pct,cm_pct
+"""
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+import numpy as np
+
+
+# ── encoding ──────────────────────────────────────────────────────────────────
+
+_ENCODE = {'A': 0, 'C': 1, 'G': 2, 'T': 3,
+           'a': 0, 'c': 1, 'g': 2, 't': 3}
+
+_DECODE = ['A', 'C', 'G', 'T']
+
+
+def encode_kmer(s: str) -> int:
+    kmer = 0
+    for c in s:
+        kmer = (kmer << 2) | _ENCODE[c]
+    return kmer
+
+
+def decode_kmer(val: int, k: int) -> str:
+    bases = []
+    for _ in range(k):
+        bases.append(_DECODE[val & 3])
+        val >>= 2
+    return ''.join(reversed(bases))
+
+
+# ── single-pass dump ──────────────────────────────────────────────────────────
+
+def stream_merged_dump(obikmer_bin: str, index_dir: str,
+                       ) -> tuple[list[str], dict[str, tuple[list[int], list[int]]]]:
+    """Stream the merged dump once.
+
+    Returns:
+        specimen_names : column labels in dump order
+        per_specimen   : mapping label → (kmer_ints, counts) for entries > 0
+    """
+    cmd = [obikmer_bin, 'dump', index_dir]
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+                            text=True)
+
+    header_line = proc.stdout.readline().rstrip('\n')
+    cols = header_line.split(',')
+    specimen_names = cols[1:]
+    per_specimen: dict[str, tuple[list[int], list[int]]] = {
+        name: ([], []) for name in specimen_names}
+
+    for line in proc.stdout:
+        parts = line.rstrip('\n').split(',')
+        kmer_int = encode_kmer(parts[0])
+        for i, name in enumerate(specimen_names):
+            count = int(parts[i + 1])
+            if count > 0:
+                per_specimen[name][0].append(kmer_int)
+                per_specimen[name][1].append(count)
+
+    proc.wait()
+    if proc.returncode != 0:
+        print(f'ERROR: obikmer dump exited {proc.returncode}', file=sys.stderr)
+        sys.exit(1)
+
+    return specimen_names, per_specimen
+
+
+# ── per-specimen comparison ───────────────────────────────────────────────────
+
+def compare_specimen(name: str,
+                     kmer_list: list[int],
+                     count_list: list[int],
+                     ref_dir: Path,
+                     k: int,
+                     save_fn: Path | None,
+                     save_fp: Path | None,
+                     save_cm: Path | None,
+                     ) -> str:
+    ref_path = ref_dir / f'{name}.npz'
+    if not ref_path.exists():
+        print(f'  SKIP {name}: no reference at {ref_path}', file=sys.stderr)
+        return ''
+
+    species = name.split('--')[0]
+    strain  = name[len(species) + 2:]
+
+    npz        = np.load(ref_path)
+    ref_kmers  = npz['kmers']    # sorted uint64
+    ref_counts = npz['counts']   # uint32
+
+    order      = np.argsort(np.array(kmer_list, dtype=np.uint64), kind='stable')
+    idx_kmers  = np.array(kmer_list,  dtype=np.uint64)[order]
+    idx_counts = np.array(count_list, dtype=np.uint32)[order]
+
+    false_neg = np.setdiff1d(ref_kmers, idx_kmers, assume_unique=True)
+    false_pos = np.setdiff1d(idx_kmers, ref_kmers, assume_unique=True)
+
+    # Count mismatches among shared kmers
+    pos_in_idx     = np.searchsorted(idx_kmers, ref_kmers)
+    pos_in_idx     = np.clip(pos_in_idx, 0, len(idx_kmers) - 1)
+    shared_mask    = idx_kmers[pos_in_idx] == ref_kmers
+    mismatch_mask  = ref_counts[shared_mask] != idx_counts[pos_in_idx[shared_mask]]
+    cm_kmers       = ref_kmers[shared_mask][mismatch_mask]
+    cm_ref         = ref_counts[shared_mask][mismatch_mask]
+    cm_idx         = idx_counts[pos_in_idx[shared_mask]][mismatch_mask]
+
+    n_shared = int(shared_mask.sum())
+    fn_pct   = 100.0 * len(false_neg) / len(ref_kmers) if len(ref_kmers) else 0.0
+    fp_pct   = 100.0 * len(false_pos) / len(idx_kmers) if len(idx_kmers) else 0.0
+    cm_pct   = 100.0 * len(cm_kmers)  / n_shared       if n_shared        else 0.0
+
+    print(f'  {name}: ref={len(ref_kmers):,}  idx={len(idx_kmers):,}  '
+          f'fn={len(false_neg):,} ({fn_pct:.4f}%)  '
+          f'fp={len(false_pos):,} ({fp_pct:.4f}%)  '
+          f'cm={len(cm_kmers):,} ({cm_pct:.4f}%)',
+          file=sys.stderr)
+
+    if save_fn and len(false_neg):
+        fn_file = save_fn / f'{name}_fn.txt'
+        fn_file.write_text('\n'.join(decode_kmer(int(v), k) for v in false_neg) + '\n')
+
+    if save_fp and len(false_pos):
+        fp_file = save_fp / f'{name}_fp.txt'
+        fp_file.write_text('\n'.join(decode_kmer(int(v), k) for v in false_pos) + '\n')
+
+    if save_cm and len(cm_kmers):
+        cm_file = save_cm / f'{name}_cm.csv'
+        lines = ['kmer,ref_count,idx_count']
+        for v, rc, ic in zip(cm_kmers, cm_ref, cm_idx):
+            lines.append(f'{decode_kmer(int(v), k)},{rc},{ic}')
+        cm_file.write_text('\n'.join(lines) + '\n')
+
+    return (f'{species},{strain},'
+            f'{len(ref_kmers)},{len(idx_kmers)},'
+            f'{len(false_neg)},{len(false_pos)},{len(cm_kmers)},'
+            f'{fn_pct:.4f},{fp_pct:.4f},{cm_pct:.4f}')
+
+
+# ── main ─────────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument('index',     metavar='INDEX_DIR', nargs='?',
+                    help='Merged count index directory')
+    ap.add_argument('ref_dir',   metavar='REF_DIR',   nargs='?',
+                    help='Directory containing per-specimen .npz reference files')
+    ap.add_argument('--obikmer', default='obikmer')
+    ap.add_argument('--header',  action='store_true',
+                    help='Print CSV header and exit')
+    ap.add_argument('--save-fn', metavar='DIR',
+                    help='Directory for false-negative kmer lists')
+    ap.add_argument('--save-fp', metavar='DIR',
+                    help='Directory for false-positive kmer lists')
+    ap.add_argument('--save-cm', metavar='DIR',
+                    help='Directory for count-mismatch CSV files')
+    args = ap.parse_args()
+
+    if args.header:
+        print('species,strain,ref_kmers,idx_kmers,'
+              'false_neg,false_pos,count_mismatch,'
+              'fn_pct,fp_pct,cm_pct')
+        return
+
+    ref_dir = Path(args.ref_dir)
+    save_fn = Path(args.save_fn) if args.save_fn else None
+    save_fp = Path(args.save_fp) if args.save_fp else None
+    save_cm = Path(args.save_cm) if args.save_cm else None
+    for d in (save_fn, save_fp, save_cm):
+        if d: d.mkdir(parents=True, exist_ok=True)
+
+    out1 = subprocess.check_output(
+        [args.obikmer, 'dump', '--head', '1', args.index],
+        stderr=subprocess.DEVNULL, text=True)
+    k = len(out1.splitlines()[1].split(',')[0])
+
+    print(f'k={k}  streaming merged dump: {args.index}', file=sys.stderr)
+    specimen_names, per_specimen = stream_merged_dump(args.obikmer, args.index)
+    print(f'{len(specimen_names)} specimen columns loaded', file=sys.stderr)
+
+    for name in specimen_names:
+        kmers, counts = per_specimen[name]
+        row = compare_specimen(name, kmers, counts, ref_dir, k,
+                               save_fn, save_fp, save_cm)
+        if row:
+            print(row)
+
+
+if __name__ == '__main__':
+    main()

benchmark/verify_merge_count.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
+INDEX="${SCRIPT_DIR}/global_index_count"
+REF_DIR="${SCRIPT_DIR}/reference_index"
+STATS_DIR="${SCRIPT_DIR}/stats/verify_merge_count"
+PYTHON="${SCRIPT_DIR}/../.venv/bin/python3"
+VERIFY_PY="${SCRIPT_DIR}/verify_merge_count.py"
+
+mkdir -p "${STATS_DIR}"
+
+CURRENT="${STATS_DIR}/current.csv"
+
+"${PYTHON}" "${VERIFY_PY}" --header >"${CURRENT}"
+
+"${PYTHON}" "${VERIFY_PY}" \
+    --obikmer "${BINARY}" \
+    "${INDEX}" "${REF_DIR}" \
+    >>"${CURRENT}"
+
+run_n=$(printf '%03d' "$(find "${STATS_DIR}" -maxdepth 1 -name 'count_*.csv' | wc -l | tr -d ' ')")
+ARCHIVE="${STATS_DIR}/count_${run_n}.csv"
+cp "${CURRENT}" "${ARCHIVE}"
+
+echo "Done. Results → ${ARCHIVE}"

benchmark/verify_merge_presence.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""Verify the merged presence index against all per-specimen reference sets.
+
+Streams `obikmer dump` once on the merged index, accumulates per-specimen
+kmer sets from each column, then compares each against its reference .npz.
+
+Output to stdout: one CSV row per specimen (same columns as verify_presence.py)
+  species,strain,ref_kmers,idx_kmers,false_neg,false_pos,fn_pct,fp_pct
+"""
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+import numpy as np
+
+
+# ── encoding ──────────────────────────────────────────────────────────────────
+
+_ENCODE = {'A': 0, 'C': 1, 'G': 2, 'T': 3,
+           'a': 0, 'c': 1, 'g': 2, 't': 3}
+
+_DECODE = ['A', 'C', 'G', 'T']
+
+
+def encode_kmer(s: str) -> int:
+    kmer = 0
+    for c in s:
+        kmer = (kmer << 2) | _ENCODE[c]
+    return kmer
+
+
+def decode_kmer(val: int, k: int) -> str:
+    bases = []
+    for _ in range(k):
+        bases.append(_DECODE[val & 3])
+        val >>= 2
+    return ''.join(reversed(bases))
+
+
+# ── single-pass dump ──────────────────────────────────────────────────────────
+
+def stream_merged_dump(obikmer_bin: str, index_dir: str,
+                       ) -> tuple[list[str], dict[str, list[int]]]:
+    """Stream the merged dump once.
+
+    Returns:
+        specimen_names : column labels in dump order (excluding 'kmer')
+        per_specimen   : mapping label → list of kmer ints where presence > 0
+    """
+    cmd = [obikmer_bin, 'dump', index_dir]
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+                            text=True)
+
+    header_line = proc.stdout.readline().rstrip('\n')
+    cols = header_line.split(',')
+    specimen_names = cols[1:]           # first col is 'kmer'
+    per_specimen: dict[str, list[int]] = {name: [] for name in specimen_names}
+
+    for line in proc.stdout:
+        parts = line.rstrip('\n').split(',')
+        kmer_int = encode_kmer(parts[0])
+        for i, name in enumerate(specimen_names):
+            if int(parts[i + 1]) > 0:
+                per_specimen[name].append(kmer_int)
+
+    proc.wait()
+    if proc.returncode != 0:
+        print(f'ERROR: obikmer dump exited {proc.returncode}', file=sys.stderr)
+        sys.exit(1)
+
+    return specimen_names, per_specimen
+
+
+# ── per-specimen comparison ───────────────────────────────────────────────────
+
+def compare_specimen(name: str,
+                     kmer_list: list[int],
+                     ref_dir: Path,
+                     k: int,
+                     save_fn: Path | None,
+                     save_fp: Path | None,
+                     ) -> str:
+    """Compare one specimen column against its reference .npz.
+
+    Returns a CSV row string.
+    """
+    ref_path = ref_dir / f'{name}.npz'
+    if not ref_path.exists():
+        print(f'  SKIP {name}: no reference at {ref_path}', file=sys.stderr)
+        return ''
+
+    species = name.split('--')[0]
+    strain  = name[len(species) + 2:]
+
+    ref_kmers = np.load(ref_path)['kmers']          # sorted uint64
+    idx_kmers = np.array(sorted(kmer_list), dtype=np.uint64)
+
+    false_neg = np.setdiff1d(ref_kmers, idx_kmers, assume_unique=True)
+    false_pos = np.setdiff1d(idx_kmers, ref_kmers, assume_unique=True)
+
+    fn_pct = 100.0 * len(false_neg) / len(ref_kmers) if len(ref_kmers) else 0.0
+    fp_pct = 100.0 * len(false_pos) / len(idx_kmers) if len(idx_kmers) else 0.0
+
+    print(f'  {name}: ref={len(ref_kmers):,}  idx={len(idx_kmers):,}  '
+          f'fn={len(false_neg):,} ({fn_pct:.4f}%)  '
+          f'fp={len(false_pos):,} ({fp_pct:.4f}%)',
+          file=sys.stderr)
+
+    if save_fn and len(false_neg):
+        fn_file = save_fn / f'{name}_fn.txt'
+        fn_file.write_text('\n'.join(decode_kmer(int(v), k) for v in false_neg) + '\n')
+
+    if save_fp and len(false_pos):
+        fp_file = save_fp / f'{name}_fp.txt'
+        fp_file.write_text('\n'.join(decode_kmer(int(v), k) for v in false_pos) + '\n')
+
+    return (f'{species},{strain},'
+            f'{len(ref_kmers)},{len(idx_kmers)},'
+            f'{len(false_neg)},{len(false_pos)},'
+            f'{fn_pct:.4f},{fp_pct:.4f}')
+
+
+# ── main ─────────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument('index',     metavar='INDEX_DIR', nargs='?',
+                    help='Merged presence index directory')
+    ap.add_argument('ref_dir',   metavar='REF_DIR',   nargs='?',
+                    help='Directory containing per-specimen .npz reference files')
+    ap.add_argument('--obikmer', default='obikmer')
+    ap.add_argument('--header',  action='store_true',
+                    help='Print CSV header and exit')
+    ap.add_argument('--save-fn', metavar='DIR',
+                    help='Directory to save false-negative kmer lists')
+    ap.add_argument('--save-fp', metavar='DIR',
+                    help='Directory to save false-positive kmer lists')
+    args = ap.parse_args()
+
+    if args.header:
+        print('species,strain,ref_kmers,idx_kmers,'
+              'false_neg,false_pos,fn_pct,fp_pct')
+        return
+
+    ref_dir  = Path(args.ref_dir)
+    save_fn  = Path(args.save_fn) if args.save_fn else None
+    save_fp  = Path(args.save_fp) if args.save_fp else None
+    if save_fn: save_fn.mkdir(parents=True, exist_ok=True)
+    if save_fp: save_fp.mkdir(parents=True, exist_ok=True)
+
+    # Detect k
+    out1 = subprocess.check_output(
+        [args.obikmer, 'dump', '--head', '1', args.index],
+        stderr=subprocess.DEVNULL, text=True)
+    k = len(out1.splitlines()[1].split(',')[0])
+
+    print(f'k={k}  streaming merged dump: {args.index}', file=sys.stderr)
+    specimen_names, per_specimen = stream_merged_dump(args.obikmer, args.index)
+    print(f'{len(specimen_names)} specimen columns loaded', file=sys.stderr)
+
+    for name in specimen_names:
+        row = compare_specimen(name, per_specimen[name], ref_dir, k, save_fn, save_fp)
+        if row:
+            print(row)
+
+
+if __name__ == '__main__':
+    main()

benchmark/verify_merge_presence.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
+INDEX="${SCRIPT_DIR}/global_index_presence"
+REF_DIR="${SCRIPT_DIR}/reference_index"
+STATS_DIR="${SCRIPT_DIR}/stats/verify_merge_presence"
+PYTHON="${SCRIPT_DIR}/../.venv/bin/python3"
+VERIFY_PY="${SCRIPT_DIR}/verify_merge_presence.py"
+
+mkdir -p "${STATS_DIR}"
+
+CURRENT="${STATS_DIR}/current.csv"
+
+"${PYTHON}" "${VERIFY_PY}" --header >"${CURRENT}"
+
+"${PYTHON}" "${VERIFY_PY}" \
+    --obikmer "${BINARY}" \
+    "${INDEX}" "${REF_DIR}" \
+    >>"${CURRENT}"
+
+run_n=$(printf '%03d' "$(find "${STATS_DIR}" -maxdepth 1 -name 'presence_*.csv' | wc -l | tr -d ' ')")
+ARCHIVE="${STATS_DIR}/presence_${run_n}.csv"
+cp "${CURRENT}" "${ARCHIVE}"
+
+echo "Done. Results → ${ARCHIVE}"

benchmark/verify_one_count.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Usage: verify_one_count.sh SPECIMEN
+# SPECIMEN = "species--strain" (Make pattern stem)
+# Output: stats/verify_count/SPECIMEN.stats (one CSV data row, no header)
+set -euo pipefail
+
+SPECIMEN="$1"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
+PYTHON="${SCRIPT_DIR}/../.venv/bin/python3"
+VERIFY_PY="${SCRIPT_DIR}/verify_count.py"
+
+species="${SPECIMEN%%--*}"
+strain="${SPECIMEN#*--}"
+
+REF_NPZ="${SCRIPT_DIR}/reference_index/${SPECIMEN}.npz"
+INDEX_DIR="${SCRIPT_DIR}/specimen_index_count/${SPECIMEN}"
+STATS_DIR="${SCRIPT_DIR}/stats/verify_count"
+STATS_FILE="${STATS_DIR}/${SPECIMEN}.stats"
+
+mkdir -p "${STATS_DIR}"
+
+echo "[${SPECIMEN}] verifying count"
+
+"${PYTHON}" "${VERIFY_PY}" \
+    --obikmer "${BINARY}" \
+    --species "${species}" \
+    --strain  "${strain}" \
+    "${REF_NPZ}" "${INDEX_DIR}" \
+    >"${STATS_FILE}"

benchmark/verify_one_presence.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Usage: verify_one_presence.sh SPECIMEN
+# SPECIMEN = "species--strain" (Make pattern stem)
+# Output: stats/verify_presence/SPECIMEN.stats (one CSV data row, no header)
+set -euo pipefail
+
+SPECIMEN="$1"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
+PYTHON="${SCRIPT_DIR}/../.venv/bin/python3"
+VERIFY_PY="${SCRIPT_DIR}/verify_presence.py"
+
+species="${SPECIMEN%%--*}"
+strain="${SPECIMEN#*--}"
+
+REF_NPZ="${SCRIPT_DIR}/reference_index/${SPECIMEN}.npz"
+INDEX_DIR="${SCRIPT_DIR}/specimen_index_presence/${SPECIMEN}"
+STATS_DIR="${SCRIPT_DIR}/stats/verify_presence"
+STATS_FILE="${STATS_DIR}/${SPECIMEN}.stats"
+
+mkdir -p "${STATS_DIR}"
+
+echo "[${SPECIMEN}] verifying presence"
+
+"${PYTHON}" "${VERIFY_PY}" \
+    --obikmer "${BINARY}" \
+    --species "${species}" \
+    --strain  "${strain}" \
+    "${REF_NPZ}" "${INDEX_DIR}" \
+    >"${STATS_FILE}"

benchmark/verify_presence.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""Compare an obikmer index against a reference kmer set (presence/absence).
+
+Loads the reference .npz (sorted uint64 kmers built by build_reference.py),
+streams the output of `obikmer dump`, encodes each kmer string to uint64,
+then reports false negatives and false positives using numpy set operations.
+
+Output to stdout: one CSV row
+  species, strain, ref_kmers, idx_kmers, false_neg, false_pos, fn_pct, fp_pct
+"""
+import argparse
+import subprocess
+import sys
+
+import numpy as np
+
+
+# ── encoding ──────────────────────────────────────────────────────────────────
+
+_ENCODE = {'A': 0, 'C': 1, 'G': 2, 'T': 3,
+           'a': 0, 'c': 1, 'g': 2, 't': 3}
+
+_DECODE = ['A', 'C', 'G', 'T']
+
+
+def encode_kmer(s: str) -> int:
+    kmer = 0
+    for c in s:
+        kmer = (kmer << 2) | _ENCODE[c]
+    return kmer
+
+
+def decode_kmer(val: int, k: int) -> str:
+    bases = []
+    for _ in range(k):
+        bases.append(_DECODE[val & 3])
+        val >>= 2
+    return ''.join(reversed(bases))
+
+
+# ── dump parsing ──────────────────────────────────────────────────────────────
+
+def load_index_kmers(obikmer_bin: str, index_dir: str) -> np.ndarray:
+    """Stream `obikmer dump` and return a sorted uint64 array of kmer integers."""
+    cmd = [obikmer_bin, 'dump', index_dir]
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+                            text=True)
+    kmers = []
+    header = True
+    for line in proc.stdout:
+        if header:
+            header = False
+            continue
+        kmer_str = line.split(',', 1)[0]
+        kmers.append(encode_kmer(kmer_str))
+    proc.wait()
+    if proc.returncode != 0:
+        print(f'ERROR: obikmer dump exited {proc.returncode}', file=sys.stderr)
+        sys.exit(1)
+    arr = np.array(kmers, dtype=np.uint64)
+    arr.sort()
+    return arr
+
+
+# ── comparison ────────────────────────────────────────────────────────────────
+
+def compare(ref: np.ndarray, idx: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+    """Return (false_negatives, false_positives) as uint64 arrays."""
+    false_neg = np.setdiff1d(ref, idx, assume_unique=True)
+    false_pos = np.setdiff1d(idx, ref, assume_unique=True)
+    return false_neg, false_pos
+
+
+# ── main ─────────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument('reference',  metavar='REF_NPZ',   nargs='?', help='Reference .npz file')
+    ap.add_argument('index',      metavar='INDEX_DIR', nargs='?', help='obikmer index directory')
+    ap.add_argument('--obikmer',  default='obikmer',   help='Path to obikmer binary')
+    ap.add_argument('--species',  default='',          help='Species label for CSV row')
+    ap.add_argument('--strain',   default='',          help='Strain label for CSV row')
+    ap.add_argument('--header',   action='store_true', help='Print CSV header and exit')
+    ap.add_argument('--save-fp',  metavar='FILE',
+                    help='Save false-positive kmer strings to FILE')
+    ap.add_argument('--save-fn',  metavar='FILE',
+                    help='Save false-negative kmer strings to FILE')
+    args = ap.parse_args()
+
+    if args.header:
+        print('species,strain,ref_kmers,idx_kmers,'
+              'false_neg,false_pos,fn_pct,fp_pct')
+        return
+
+    # Detect k from the index (one cheap call before the full dump).
+    cmd1 = [args.obikmer, 'dump', '--head', '1', args.index]
+    out1 = subprocess.check_output(cmd1, stderr=subprocess.DEVNULL, text=True)
+    k = len(out1.splitlines()[1].split(',')[0])
+
+    # Load reference
+    print(f'Loading reference: {args.reference}', file=sys.stderr)
+    npz = np.load(args.reference)
+    ref_kmers = npz['kmers']          # already sorted uint64
+
+    # Load index
+    print(f'Streaming dump (k={k}): {args.index}', file=sys.stderr)
+    idx_kmers = load_index_kmers(args.obikmer, args.index)
+
+    print(f'k={k}  ref={len(ref_kmers):,}  idx={len(idx_kmers):,}', file=sys.stderr)
+
+    false_neg, false_pos = compare(ref_kmers, idx_kmers)
+
+    fn_pct = 100.0 * len(false_neg) / len(ref_kmers) if len(ref_kmers) else 0.0
+    fp_pct = 100.0 * len(false_pos) / len(idx_kmers) if len(idx_kmers) else 0.0
+
+    print(f'false negatives: {len(false_neg):,}  ({fn_pct:.4f}%)', file=sys.stderr)
+    print(f'false positives: {len(false_pos):,}  ({fp_pct:.4f}%)', file=sys.stderr)
+
+    if args.save_fn and len(false_neg):
+        with open(args.save_fn, 'w') as fh:
+            for v in false_neg:
+                fh.write(decode_kmer(int(v), k) + '\n')
+        print(f'False negatives saved → {args.save_fn}', file=sys.stderr)
+
+    if args.save_fp and len(false_pos):
+        with open(args.save_fp, 'w') as fh:
+            for v in false_pos:
+                fh.write(decode_kmer(int(v), k) + '\n')
+        print(f'False positives saved → {args.save_fp}', file=sys.stderr)
+
+    print(f'{args.species},{args.strain},'
+          f'{len(ref_kmers)},{len(idx_kmers)},'
+          f'{len(false_neg)},{len(false_pos)},'
+          f'{fn_pct:.4f},{fp_pct:.4f}')
+
+
+if __name__ == '__main__':
+    main()

docmd/architecture/numa_partition_runner.md

@@ -162,14 +162,158 @@ A single `PartitionRunner` instance can be built once per command invocation
 and reused across multiple `run()` calls (e.g. `merge` runs
 `merge_partitions` then `pack_matrices`).
 
+## Known issue: CPU-only activation signal stalls on I/O-bound stages
+
+Observed on a real `filter` run (109 genomes, 256 partitions, 8×24-core NUMA):
+`rebuild` (CPU-bound — k-mer construction) scales cleanly from 9 to 43 active
+workers as `CpuSample::do_i_activate` (`obisys::lib.rs`) sees efficiency climb.
+`pack_matrices` (I/O-bound — reopens and recomposes per-genome column files
+into `.pbmx`/`.pcmx`) activates one extra worker then flatlines at 10/192 for
+the rest of the stage, even though 256 partitions keep completing over several
+minutes. This matches the documented intent (§ Adaptive mechanism — "avoids
+over-provisioning ... I/O-bound ... workloads") but conflates two different
+things: *"CPU is not the bottleneck"* and *"more workers would not help"*. On
+storage with real queue depth (NVMe, RAID, parallel FS) the second stage could
+still benefit from more concurrent workers even with flat CPU usage — a signal
+the current mechanism cannot see.
+
+A one-off artefact was also found in the same log: right after a stage
+transition, `do_i_activate` produced a physically impossible spike (efficiency
+~94 cores on a 192-core box) because it has no minimum-window guard — unlike
+its sibling `cpu_efficiency`, which returns `0.0` if `wall < 0.1s`
+(`obisys::lib.rs:260`). `do_i_activate` unconditionally overwrites
+`self.wall`/`self.user_secs`/`self.sys_secs` even when the elapsed window is
+too short to be meaningful, so a burst of rapid completions right after
+activating a worker can divide a real CPU delta by a near-zero wall delta.
+
+### Implemented: I/O signal + shared debounce guard
+
+`IoSample` (`obisys::lib.rs`, alongside `CpuSample`) is fed by
+`read_bytes`/`write_bytes` from `/proc/self/io` on Linux (actual bytes
+submitted to the block layer — not `rchar`/`wchar`, which also count
+page-cache hits, and not `ru_inblock`/`ru_oublock`, unreliable on macOS), with
+a `proc_pid_rusage(RUSAGE_INFO_V4)` fallback on macOS
+(`ri_diskio_bytesread`/`ri_diskio_byteswritten`, FFI only via `libc`, no new
+dependency — same pattern as the existing `getrusage` bindings). Any other
+target degrades gracefully to a signal that never triggers (falls back to
+CPU-only activation), same pattern as `cgroup_v2_available`.
+
+`maybe_activate` (`numa.rs`) activates a worker if *either* signal still shows
+headroom, making `PartitionRunner` adapt to whichever resource is actually the
+bottleneck without per-call configuration. Both samplers are called
+unconditionally — no `||` short-circuit — so neither window starves behind
+whichever signal fires first:
+
+```rust
+let cpu_threshold = CPU_SPAWN_THRESHOLD * activation.last_step() as f64;
+let cpu_wants_more = cpu_sample.do_i_activate(cpu_threshold);
+let io_wants_more  = io_sample.do_i_activate(IO_SPAWN_THRESHOLD);
+if cpu_wants_more || io_wants_more {
+    activation.grow(GROWTH_DIVISOR, n_total);
+}
+```
+
+The CPU threshold is *not* the flat absolute delta it started as: it scales
+with `activation.last_step()` — the number of workers activated in the last
+growth step, tracked by `NodeActivation` (`numa.rs`) and updated every time
+`grow()` actually grows something. Growing by 8 workers should add ~8 cores of
+efficiency if the workload is truly CPU-bound; requiring only
+`CPU_SPAWN_THRESHOLD` (20 %) of that expected gain confirms the growth was
+useful without demanding perfect linear scaling. Scaling by the *last step's
+size* rather than the cumulative total keeps the bar equally meaningful
+whether it's the 2nd growth step or the 20th — a flat absolute threshold
+(0.2 core) is a strong signal at 8 active workers but pure noise at 150; a
+threshold scaled by the *cumulative* total instead (considered and rejected)
+would have made the bar essentially impossible to clear late in the ramp,
+strangling exactly the CPU-bound saturation the mechanism exists to allow.
+
+Unlike the CPU signal (an absolute delta in cores — a bounded, portable unit),
+raw I/O throughput has no natural scale across devices, so `IoSample` uses a
+**relative** growth threshold instead of an absolute one:
+
+```rust
+pub fn do_i_activate(&mut self, threshold: f64) -> bool {
+    let elapsed = self.wall.elapsed().as_secs_f64();
+    if elapsed < 0.1 { return false; }        // state untouched — window keeps accumulating
+
+    let n = Self::read_bytes();
+    let rate = n.saturating_sub(self.bytes) as f64 / elapsed;
+    let activate = if self.previous_rate == 0.0 {
+        rate > 0.0                            // bootstrap: any measured throughput is signal
+    } else {
+        (rate - self.previous_rate) / self.previous_rate >= threshold
+    };
+
+    self.bytes = n;
+    self.wall  = Instant::now();              // reset only on a real sample
+    activate
+}
+```
+
+The `elapsed < 0.1s → return false without mutating state` guard was also
+back-ported into `CpuSample::do_i_activate` (previously missing — source of
+the ~94-core artefact above) — one fix for both problems, and it removes the
+need for any arbitrary I/O-rate floor: a short/noisy window is rejected
+outright rather than papered over with a hardware-dependent constant.
+
+Both spawn thresholds (`CPU_SPAWN_THRESHOLD`, `IO_SPAWN_THRESHOLD`, module-level
+`const` in `numa.rs`, both `0.2`) are a starting point, not a derived value:
+`0.2` (20 % relative growth) for `IoSample` was chosen to match the CPU
+threshold's *implicit* relative sensitivity (in the observed log, an 8→9
+worker step raised efficiency by ~12 %) — but I/O throughput is lumpier than
+CPU time (buffered writes flush in bursts), so it needs empirical validation
+against a real `pack` run before being considered final.
+
+## Known issue: ramp-up too slow, and confused with node count
+
+The original design started `n_nodes` workers (one per node) and grew one
+worker at a time. On a real `filter` run this took ~10 minutes to climb from
+9 to ~40 active workers even on the CPU-bound `rebuild` stage — most of a
+35-minute stage spent under-provisioned while waiting for evidence to
+accumulate one worker at a time. There is no scale-down mechanism (`n_active`
+only grows), so the original caution was deliberate — but a quarter of
+available cores is still far from saturation, and the real risk zone (over-provisioning
+a memory-bandwidth-bound stage) only shows up much later in the ramp, near
+full occupancy — not at 25 %.
+
+The fix decouples ramp speed from node *count*: both the initial size and the
+growth step are a fraction of `workers_per_node` (node *size*), applied
+identically on every node. A single-NUMA-node (UMA) machine ramps exactly as
+fast as an 8-node one — growing by `n_nodes` per step, as first considered,
+would have degenerated to "grow by 1" on UMA, reproducing the original
+problem for exactly the machines that need the fix most.
+
+```rust
+// NodeActivation::grow — called both at startup (activate_initial) and on
+// every CPU/IO-triggered growth step, with a different divisor each time.
+let wanted = (self.caps[idx] / divisor).max(1);   // INITIAL_DIVISOR=4 at startup, GROWTH_DIVISOR=8 per step
+let room   = self.caps[idx].saturating_sub(self.active[idx]);
+let grow   = wanted.min(room).min(n_total.saturating_sub(self.total));
+```
+
+This also fixed a latent correctness gap: the original single shared
+`activate_tx`/`activate_rx` pair had *no* per-node addressing — sending one
+activation signal woke up whichever dormant worker (from any node) happened
+to win the race on that channel. `crossbeam_channel` gives no fairness
+guarantee across competing receivers, so "round-robin across nodes" was an
+assumption the code never actually enforced. `PartitionRunner::run` now opens
+one activation channel per node (`activate_txs`/`activate_rxs`, one pair per
+`NodeConfig`); `NodeActivation` (`numa.rs`) tracks how many of each node's
+dormant workers have been woken and grows every node by the same amount per
+step, capped by that node's remaining dormant workers and by the run's total
+budget (`n_total`) — balance across nodes is now guaranteed by construction,
+not incidental to channel implementation details.
+
 ## Open questions
 
 - **Error handling**: `run` currently returns the first error; remaining errors
   are dropped.  A `Vec<E>` return would give complete diagnostics.
 
-- **`workers_per_node` tuning**: currently `(cpus / 8).max(3).min(8)`, calibrated
-  for merge on BeeGFS.  I/O-bound commands (`dump`, `select`) may benefit from
-  a higher value.  A per-call override could be added to the API.
+- **`INITIAL_DIVISOR` / `GROWTH_DIVISOR` tuning**: currently `4` and `8`
+  (start at 1/4 of a node's cores, grow by 1/8 per step), chosen to fix an
+  observed too-slow ramp — not yet validated against a real `pack` (I/O-bound)
+  run, where over-provisioning risk is different from the CPU-bound `rebuild`
+  case this was tuned against.
 
 - **`on_done` ordering**: the runner serialises calls to `on_done` via an
   internal `Arc<Mutex<C>>`.  `Send` is required (the Arc clone crosses thread

docmd/architecture/query.md

@@ -16,27 +16,43 @@ Given a set of query sequences, determine for each sequence how many of its k-me
 
 ## Algorithm
 
-The query follows the same superkmer-based partitioning strategy used at indexing time.
+The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside `process_chunk` (`query.rs`); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (`obikpartitionner::query_partition_with`), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.
 
 ```
-for each chunk of sequences (parallel workers via obipipeline):
-    build QueryBatch: decompose all sequences into s-mers via superkmers, deduplicate
-    allocate seq_results[seq_idx][smer_pos] = None  ← per-sequence s-mer result vectors
-    split superkmers by partition via minimiser hash
+for each chunk of sequences (parallel workers via obipipeline, one call to process_chunk):
+    build QueryBatch (QueryBatch::from_records):
+        decompose all sequences into superkmers (SuperKmerIter) — construction only,
+        not the dedup key
+        deduplicate at k-mer granularity, split by partition in the same pass:
+            by_partition: Vec<HashMap<CanonicalKmer, Vec<KmerDesc>>>   ← KmerDesc = (seq_idx, pos)
+    allocate SmerIndex (SmerIndex::new): in_index: Vec<bool>, sized total_smers —
+        NOT multiplied by n_genomes
+    allocate by_genome: Vec<Vec<(seq_idx, pos, value)>>, one empty Vec per genome —
+        stays empty (zero cost) for every genome this chunk never matches
     for each partition p:
-        query_partition(p, superkmers_routed_to_p)
-            → load QueryLayer(s) for p
-            → for each s-mer in each superkmer: MphfLayer::find(smer)
-        fill seq_results[seq_idx][kmer_offset + j] from partition results
-    for each sequence:
-        apply_findere(seq_results[seq_idx], effective_z)  ← per full sequence
-        accumulate confirmed k-mer results into acc and cov
-    emit annotated sequences
+        query_partition_with(p, kmers_for_p, on_event):
+            stage 1 (MPHF-only): for each unique k-mer, try each layer's MphfLayer::find
+                in turn, stop at the first hit; bucket confirmed hits by (layer, slot);
+                emit QueryHit::Found(descs) once per hit k-mer
+            stage 2 (column-major fetch): for each layer with ≥1 hit, for each genome
+                column g in 0..layer.n_cols(): scan that layer's bucketed slots, look up
+                col_value(g, slot); emit QueryHit::Value(descs, g, value) on nonzero
+        on_event dispatches: Found → SmerIndex::mark_found for every desc;
+                              Value → push (seq_idx, pos, value) into by_genome[g]
+    for each genome g with ≥1 hit (sparse_findere_for_genome):
+        sort by_genome[g] by (seq_idx, pos); detect maximal runs of consecutive pos
+        within one seq_idx; monotone-deque window-minimum scoped to each run →
+        confirmed_by_genome[g]: Vec<(seq_idx, pos_out, value)>
+    accumulate genome_totals per sequence from confirmed_by_genome (per genome, direct)
+    accumulate kmer_count / kmer_missing per (sequence, output position), O(1) each,
+        using only the confirmed-any bitmap and SmerIndex — independent of n_genomes
+    if --detail: densify confirmed_by_genome into per-(seq, genome) coverage arrays
+    emit annotated sequences (emit_batch)
 ```
 
-Superkmers that appear more than once in the batch (same sequence or across sequences) are deduplicated: each unique `RoutableSuperKmer` is queried once per partition, and the result is broadcast to every `SKDesc` entry that references it.
+Superkmers that appear more than once in the batch (same sequence or across sequences), or different superkmers that happen to share a k-mer (read overlaps, repeats, a SNP splitting an otherwise-identical run), are deduplicated at k-mer granularity: each unique `CanonicalKmer` triggers at most one MPHF lookup and, on hit, one matrix fetch, broadcast to every `KmerDesc` occurrence referencing it.
 
-**Findere requires full-sequence aggregation.** `apply_findere` is applied once per sequence on the complete s-mer result vector, after all partitions have contributed. Applying it per superkmer would produce false negatives at superkmer boundaries, where the z-window spans two superkmers.
+**Findere requires full-sequence aggregation.** The sliding window (now per-run, not per-sequence — see [Findere z-window filter](#findere-z-window-filter)) only ever runs after all partitions have contributed their hits to `by_genome`. Applying it per superkmer would produce false negatives at superkmer boundaries, where the z-window spans two superkmers.
 
 Batches are processed in parallel via `obipipeline` workers; the `--threads` flag controls the number of worker threads.
 
@@ -44,25 +60,46 @@ Batches are processed in parallel via `obipipeline` workers; the `--threads` fla
 
 ## Findere z-window filter
 
-For approximate index modes, the index physically stores s-mers of size `s = k_user − z + 1`. At query time, `set_k(s)` is in effect, so queries naturally produce s-mer results. `apply_findere` then aggregates z consecutive s-mer results into one k_user-mer answer:
+For approximate index modes, the index physically stores s-mers of size `s = k_user − z + 1`; `idx.kmer_size()` (bound to `k` in `process_chunk`) is this physically-indexed s-mer size, so decomposing the query at `k` naturally produces s-mer results.
 
-```rust
-fn apply_findere(
-    results: &[Option<Box<[u32]>>],  // N s-mer results
-    z: usize,
-    n_genomes: usize,
-) -> Vec<Option<Box<[u32]>>>         // N − z + 1 k_user-mer results
+The z-window aggregation is **sparse**, per genome, implemented in `sparse_findere_for_genome` (`query.rs`) — a run-detection pass followed by a monotone-deque sliding-window minimum scoped to each run, not a dense scan over every s-mer position of every sequence:
+
+```
+sparse_findere_for_genome(hits, z, presence, threshold):
+    // hits: raw (seq_idx, pos_smer, value) triples for this genome, as delivered
+    // by query_partition_with's QueryHit::Value — only ever nonzero entries;
+    // a position with no hit for this genome simply has no entry at all.
+    sort hits by (seq_idx, pos_smer)
+
+    for each maximal run of consecutive pos_smer values within the same seq_idx:
+        dq: VecDeque<(run-relative index, value)>
+        for k, (_, pos, value) in enumerate(run):
+            maintain dq monotone non-decreasing (pop back while back.value >= value)
+            push (k, value)
+            evict dq entries with run-relative index <= k - z
+            if k + 1 >= z:
+                win_min = dq.front().value
+                if win_min > 0:
+                    pos_out = pos + 1 - z
+                    confirmed.push((seq_idx, pos_out, adjust(win_min)))
+    return confirmed
 ```
 
-Input length N (s-mers), output length N − z + 1 (k_user-mers).
+A window can only be confirmed (`win_min > 0`) when all `z` s-mers in it are present *and* nonzero for this genome — which, by construction, can only happen strictly inside one contiguous run of hits (any gap — an absent or zero-valued s-mer — forces `win_min = 0` for every window spanning it, exactly matching the old dense scan's "not in index counts as 0" rule, just never materialising the zero). The deque logic is otherwise identical to the pre-sparsification version; it's scoped to run-relative indices instead of the whole sequence.
 
-For each genome g independently, a sliding window of size z scans the input. Output position i is confirmed for genome g iff all z values `results[i..i+z][g]` are nonzero (`None` counts as zero for all genomes). The scan is O(n) per genome.
+This runs once per genome that has at least one hit in the chunk (`process_chunk` iterates `by_genome`, one `Vec<(seq_idx, pos_smer, value)>` per genome, built from `QueryHit::Value` during the partition loop — genomes with zero hits in this chunk have an empty `Vec` and cost nothing beyond the iteration itself). Total work is `O(hits log hits)` per genome (the sort) rather than `O(n_smers)` per genome regardless of hit count — a genuine complexity win on top of the memory one, for the common case where most `(chunk, genome)` pairs have no or few hits.
 
-Output values come from `results[i]` (leftmost s-mer of each window); genomes not confirmed are zeroed. If all genomes are zero, the position is returned as `None`.
+Output position `pos_out` is confirmed for genome `g` iff its run produced a nonzero `win_min` — equivalent to "all `z` consecutive s-mer values in the window are nonzero for `g`", same semantics as before.
 
-**Short sequences**: when the s-mer count is less than z, no complete window can form — `apply_findere` returns an empty vector. K-mers from sequences shorter than k_user are not emitted.
+**The value reported per confirmed position is the window minimum, not the leftmost s-mer's raw value** — unchanged from the dense version. For presence indexes (0/1 values) this is equivalent to a logical AND either way. For count indexes it is not: the accumulated count for genome `g` at position `pos_out` is the minimum across the window, the weakest link — not the leftmost s-mer's own count. The presence/count adjustment (`u32::from(win_min >= threshold)` vs. raw `win_min`) is applied once, inside `sparse_findere_for_genome`, rather than later during accumulation.
 
-**Exact indexes**: `z = 1`, `apply_findere` is a passthrough (output length = input length).
+**`kmer_missing` bookkeeping is independent of the per-genome sparse structures**, by design (see roadmap point 9): a lightweight dense `SmerIndex` (`in_index: Vec<bool>`, sized `total_smers` — **not** multiplied by `n_genomes`) is populated from `QueryHit::Found` during the partition loop, one entry per hit k-mer regardless of which genome(s) it matched. A position with no genome confirmed counts as `kmer_missing` iff the leftmost s-mer of that window is absent from `SmerIndex` entirely (see [`kmer_missing` semantics](#kmer_missing-semantics)).
+
+**Coverage (`--detail`)** is built by re-scanning each genome's confirmed-hit list (already computed, no extra pass over raw data) and densifying into the `[u32; n_kmers_out]` arrays the JSON output format requires — but only when `--detail` is actually requested; the sparse structures cost nothing extra when it isn't.
+
+**Short sequences**: when a sequence's s-mer count is less than `z`, its run(s) — if any hits exist at all — can never reach length `z`, so no window is ever confirmed for it; no k_user-mer is emitted, same outcome as the dense version's `n_kmers_out == 0` early-skip, reached here as a natural consequence rather than a separate check.
+
+**Exact indexes**: `z = 1`, every single-hit "run" of length 1 immediately satisfies `k + 1 >= z`, so every hit is its own confirmed window with `win_min` equal to its own value — a passthrough, as before.
 
 ### Effective z at query time
 
@@ -85,14 +122,17 @@ The `-z` CLI option overrides the index metadata value. A higher z increases str
 
 ### `QueryLayer` variant selection
 
-`QueryLayer::open` in `query_layer.rs` selects the data matrix to pair with `MphfLayer`:
+`QueryLayer::open` (`obikpartitionner/src/query_layer.rs:28-45`) only ever returns two variants — `Presence` or `Count`, checked in this order:
 
-| Condition | Variant | Data returned per k-mer |
-|---|---|---|
-| `with_counts=true` and `counts/` exists | `Count` | raw count per genome |
-| `presence/` exists | `Presence` | 0/1 per genome (bit matrix) |
-| only `counts/` exists | `Count` | counts used as-is |
-| neither exists | `SetOnly` | 1 for every genome |
+| Order | Condition | Variant | Data returned per k-mer |
+|---|---|---|---|
+| 1 | `with_counts=true` and `counts/` exists | `Count` | raw count per genome |
+| 2 | (else) `presence/` exists, or `counts/` doesn't exist at all | `Presence` | see below |
+| 3 | (else — `counts/` exists, `presence/` doesn't, `with_counts=false`) | `Count` | counts used as-is |
+
+There is no `QueryLayer::SetOnly` variant. The "no on-disk matrix at all" case is handled one level down: `Presence` wraps `PersistentBitMatrix`, whose own `open()` (`obicompactvec/src/bitmatrix.rs:260-288`) auto-detects among **three** internal representations — `Packed` (`presence/matrix.pbmx`), `Columnar` (`presence/meta.json`), or `Implicit { n_rows, n_cols }` when neither file exists (built from `layer_meta.json`, `fill_row` returning all-`1`s without touching disk). This is where "1 for every genome" actually happens — not at the `QueryLayer` level.
+
+**Worth double-checking, not confirmed as a bug**: `PersistentBitMatrix::open`'s `Implicit` branch constructs `Implicit { n_rows: meta.n, n_cols: 1 }` — `n_cols` is hardcoded to `1`, not to the layer's actual `n_genomes`. `fill_row` for `Implicit` only writes `buf[..1]`, leaving the rest of a longer `n_genomes`-sized buffer untouched (zeroed by the caller beforehand). If this path is ever reached for a layer covering more than one genome, only genome index 0 would read as present. Whether that's reachable in practice (layers might always be single-genome when they fall back to `Implicit`) wasn't verified here — flagging for follow-up, not fixing.
 
 ---
 
@@ -123,7 +163,7 @@ Coverage reflects confirmed k_user-mers only. The vectors are emitted in the JSO
 
 ## `kmer_missing` semantics
 
-`kmer_missing` counts k_user-mer positions where the first s-mer (`seq_results[seq_idx][pos]`) is `None` — i.e. absent from the index entirely. K-mers where the z-window fails because a later s-mer is absent or zero are not counted as missing (the first s-mer being present is used as proxy for index membership).
+`kmer_missing` counts k_user-mer positions where the leftmost s-mer of the window (`smer_index.is_in_index(seq_idx, pos)`, `SmerIndex`) is `false` — i.e. absent from the index entirely. K-mers where the z-window fails because a later s-mer is absent or zero (but the leftmost one is present) are not counted as missing — the leftmost s-mer being present is used as proxy for index membership.
 
 ---
 
@@ -152,8 +192,8 @@ Genome keys follow the iteration order of `meta.genomes`.
 | Key | Type | Condition | Semantics |
 |---|---|---|---|
 | `kmer_count` | int | always | k-mers confirmed (post-Findere) with at least one genome match |
-| `kmer_missing` | int | `--count-missing` | k-mers absent from the index entirely (pre-Findere None) |
-| `kmer_strict_matches` | object | always | per-genome accumulated value (label → count or 0/1) |
+| `kmer_missing` | int | `--count-missing` | k-mers absent from the index entirely (leftmost s-mer of the window not found) |
+| `kmer_strict_matches` | object | always | per-genome accumulated value, non-zero entries only (label → count or 0/1) |
 | `coverage` | object | `--detail` | per-genome array of per-position contributions (label → [u32]) |
 
 `kmer_count + kmer_missing` ≤ total k_user-mers in the sequence. The gap corresponds to k_user-mers whose z-window was not fully confirmed (at least one s-mer absent or zero for all genomes) but whose first s-mer was present in the index.
@@ -165,7 +205,7 @@ Genome keys follow the iteration order of `meta.genomes`.
 ```
 obikmer query <index> [--detail] [--mismatch] [--count-missing]
               [--force-presence] [--presence-threshold <n>]
-              [-z <z>] [-T <threads>]
+              [-z <z>] [-T <threads>] [--chunk-size <MiB>]
               <query.fa> [<query2.fa> ...]
 ```
 
@@ -177,6 +217,7 @@ obikmer query <index> [--detail] [--mismatch] [--count-missing]
 | `--force-presence` | off | Report 0/1 per genome regardless of index counts |
 | `--presence-threshold` | 1 | Minimum count to declare genome present |
 | `-T` / `--threads` | all CPUs | Worker threads |
+| `--chunk-size` | auto (from available RAM and thread count) | I/O chunk size in MiB — see [Future work, point 3](#throughput--parallelism--identified-potential-not-yet-implemented) for why the auto-sizing formula currently under-estimates memory on indexes with many genomes |
 
 `--mismatch` is accepted but currently ignored with a warning on stderr.
 
@@ -187,3 +228,179 @@ obikmer query <index> [--detail] [--mismatch] [--count-missing]
 - **`--mismatch`**: 1-mismatch approximate matching — generate `3·k` single-substitution variants per k-mer, look each up independently.
 - **Read classification** (`--classify`): assign each read to the genome with the highest match score.
 - **Whitelist / blacklist filtering**: threshold-based accept/reject on per-genome match scores.
+
+### Throughput & parallelism — identified potential (not yet implemented)
+
+Observed on a 192-core (8×24 NUMA) machine: `query` uses ~10 cores or fewer, and the default chunk size gets the process OOM-killed. Root causes and candidate fixes, in dependency order:
+
+**1. Single-threaded I/O source (main core-utilization bottleneck).**
+`run()` builds `all_chunks` via `paths.into_iter().flat_map(read_sequence_chunks_sized(...))` and passes it directly as the `input` iterator to `pipe.apply()`. In `obipipeline::Pipe::apply` (`scheduler.rs`), `input.next()` is called exclusively from the dedicated source thread — so file opening, decompression, and FASTA/FASTQ chunk-boundary parsing for *all* input files run serially in one thread, regardless of `--threads`. Compare with `steps::scatter` (used by `index`) and `cmd/superkmer.rs`: there, file opening + streaming is itself a `Flat` pipeline stage (`||?`), executed across the `n_workers` pool, with `obipipeline::throttle(paths, max_open)` bounding concurrently-open files in the source thread. That pattern parallelises I/O across files (and NUMA nodes); `query.rs` cannot.
+Fix direction: restructure `query`'s pipe with an initial `Flat` stage analogous to `scatter`'s, opening/chunking files across workers instead of in `flat_map`.
+
+**2. Gzip decompression is inherently single-threaded per file.**
+`niffler`/`flate2` (used by `xopen`) do standard DEFLATE, which has no parallel-decodable structure for an arbitrary stream. Fix (1) parallelises *across* files but not *within* one large gzip file. Parking a possible fix (`rapidgzip-rs`) is tracked in [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen).
+
+**3. Chunk-size memory formula ignores `n_genomes`.**
+`chunk_bytes = available_memory_bytes() / (n_workers * 16)` (`query.rs:407-414`) assumes a fixed ~8–16× overhead per raw input byte. But `KmerResults::new` (`query.rs:165-179`) allocates `data: Vec<u32>` sized `total_kmers_in_chunk × n_genomes` — dense, **for every k-mer position in the chunk, hit or not** — plus `win_min` and (with `--detail`) `cov`, same scaling. Real per-chunk memory is `O(n_genomes)`, not constant; the formula doesn't know `n_genomes` at all. This is the direct cause of the OOM kill on indexes with many reference genomes.
+
+**4. MPHF lookup and matrix-row fetch are fused, not staged.**
+`QueryLayer::find_into` (`obikpartitionner/src/query_layer.rs:48-67`) does the MPHF `find` *and* the `fill_row` matrix read in one call per k-mer, inside a single-threaded loop (`query_partition_with`). There is no separation between "is this k-mer indexed" (cheap, `O(1)`, independent of `n_genomes`) and "what are its per-genome values" (the expensive, `n_genomes`-scaling part).
+
+**5. Dereplication should happen at k-mer granularity, directly — not via an intermediate superkmer-level dedup.**
+`QueryBatch::from_records` currently dereplicates at the *superkmer* level (`HashMap<RoutableSuperKmer, Vec<SKDesc>>`, `query.rs:112`). This misses redundancy between k-mers shared by *different* superkmers (read overlaps, repeats, a SNP splitting an otherwise-identical run). Superkmer *construction* (`SuperKmerIter`) stays mandatory — it is the mechanism that computes minimizers/partition routing, not an optional dedup layer — but the dedup structure built on top of it should key directly on `CanonicalKmer`, in the same pass: `HashMap<CanonicalKmer, Vec<(seq_idx, pos)>>`. This also means the MPHF `find` itself runs once per **distinct** k-mer instead of once per occurrence — a win independent of the matrix-fetch cost below.
+
+**6. Stage 1 output: bucket confirmed hits by layer, keyed by MPHF slot.**
+For each unique canonical k-mer, MPHF lookup across a partition's layers stops at the first match (`query_partition_with:105-111`) — a k-mer belongs to at most one layer. So stage 1's output can be reshaped directly into:
+```
+HashMap<layer_idx, HashMap<slot, Vec<(seq_idx, pos)>>>
+```
+replacing the `CanonicalKmer` key by the resolved `slot` (compact integer, and exactly what stage 2 needs to address the matrix). K-mers matching no layer simply have no entry here (they still count toward `in_index`/`kmer_missing` bookkeeping, which stays `O(1)` per position, independent of `n_genomes`).
+
+**7. Partition-level parallelism is currently absent — and a NUMA-aware mechanism for exactly this already exists, unused, in `obikindex`.**
+`process_chunk`'s partition loop (`query.rs:250-278`, `for (part_idx, part_sks) in by_part.iter().enumerate()`) processes every partition of a chunk sequentially on the single worker thread that owns that chunk. This is a parallelism axis on its own, independent of the column question below.
+More importantly: `docmd/architecture/numa_partition_runner.md` and `numa_worker_pools.md` document `PartitionRunner` (`obikindex/src/numa.rs`), **already implemented** and already used by `merge.rs`, `index.rs` (`build_layers`), `select.rs`, `reindex.rs`, `rebuild.rs` — one controller thread per NUMA node, a Rayon pool pinned to that node's CPUs (`hwlocality`, `numa` feature, default-on in `obikindex/Cargo.toml`), adaptive worker activation driven by *both* a CPU-efficiency signal and an I/O-throughput signal (`CpuSample`/`IoSample`, `/proc/self/io` on Linux). It exists precisely because a naive `into_par_iter()` on the global Rayon pool measurably degrades ×60 on this codebase's own 192-core/8-NUMA reference machine (`numa_worker_pools.md`, § Problem) once workers contend for cross-socket memory bandwidth on shared mmap'd/hashed structures — exactly the shape of the matrix-column scan in point 8 below.
+`obikmer` already depends on `obikindex` (`obikmer/Cargo.toml`, for `KmerIndex`), so `PartitionRunner` is directly reachable from `cmd/query.rs` — no new dependency. Both the partition-level loop and (see point 8) the genome-column scan should be driven through it rather than through ad-hoc `rayon::into_par_iter()`, to avoid reproducing the already-measured-and-fixed contention problem. Also relevant: the "CPU-only signal stalls on I/O-bound stages" issue documented for `pack_matrices` (mmap-heavy, page-fault-bound) applies just as much to a column-major mmap scan over persistent matrices — reuse the existing dual CPU/IO activation signal rather than re-deriving one.
+>
+> **Correction from implementation (Phase 4 below)**: this turned out not to be viable as described. `PartitionRunner::run()`'s actual body spawns roughly one OS thread per worker slot across every NUMA node **on every call** (confirmed by reading `numa.rs`, not just its doc comments) — fine for the one-call-per-command-invocation batch usage in `merge`/`build_layers`, but `query_partition_with` runs once per `(chunk, partition)`, far too frequently to absorb that spawn cost. Partition-level parallelism via `PartitionRunner` is deferred, not implemented. See Phase 4's "What did not ship, and why" for the detail.
+
+**8. Stage 2: column-major matrix fetch, parallel across genome columns — via `PartitionRunner`, not naive `rayon`.**
+Both persistent matrix formats are column-oriented on disk: `ColumnarCompactIntMatrix`/`ColumnarBitMatrix` (`obicompactvec/src/{intmatrix,bitmatrix}.rs`) mmap one file per genome column; `PackedCompactIntMatrix`/`PackedBitMatrix` mmap one region-offset per column in a single file. `fill_row(slot, buf)` as used today (`query.rs:262-272` via `on_hit`) reads **one slot across all `n_genomes` columns** per hit — the worst possible access pattern for this layout (up to `n_genomes` scattered mmap regions touched per single k-mer).
+Better: for each layer, walk the matrix **column by column** (genome by genome): for each genome, scan the `slot` keys collected in step 6 for that layer and call `col.get(slot)`, keeping only nonzero results, and broadcast to the associated `(seq_idx, pos)` list. Total `get()` calls are unchanged (`n_hits × n_genomes` in the worst case) — the win is locality (sequential access within one mmap'd column at a time, not scattered across all columns per hit), not fewer operations.
+Columns are independent (read-only, disjoint mmap regions) → embarrassingly parallel across genomes, *but* — per point 7 — `obicompactvec`'s existing `into_par_iter()` over `0..n_cols` (`sum()`, `count_nonzero()`, pairwise distance matrices) is the **naive, unpinned** pattern the rest of the codebase is actively migrating away from, not a model to copy here. Route this through `PartitionRunner` (or the same NUMA-pool machinery) instead. Two things to settle when this is designed: how the partition axis (point 7), the column axis, and the existing chunk-level `n_workers` `obipipeline` pool compose without oversubscribing the machine (three different concurrency mechanisms — raw-thread pipe workers, `PartitionRunner`'s pinned Rayon pools, and whatever drives the column scan — need a single reconciled thread budget, not three independent ones); and the threshold below which per-column dispatch overhead outweighs the gain (small `n_genomes` or small per-layer hit counts) — to be measured, not assumed.
+>
+> **Correction from implementation (Phase 4 below)**: column-major fetch is implemented — but as a plain sequential loop, not parallelised via `PartitionRunner`. Same reason as point 7's correction above. The column-major *locality* win (the actual claim of this point) does not depend on adding parallelism on top of it, and is validated independently. Column-level parallelism is deferred pending a mechanism that fits this call frequency (candidates noted in Phase 4).
+
+**9. Sparse per-genome representation, fed directly to Findere.**
+Stage 2's output should be `HashMap<genome_idx, Vec<(seq_idx, position, count)>>`, **sorted by `(seq_idx, position)`** once collected, instead of a dense `KmerResults`-style matrix — the key must carry `seq_idx`, not just `genome_idx`, because a chunk batches many sequences and `position` is only meaningful within one; a plain `Vec<(position, count)>` per genome would silently mix positions from different sequences and corrupt the sliding-window scan. This bounds retained memory by actual nonzero hits on both axes (position sparsity from non-matching k-mers, genome sparsity from a matched k-mer typically belonging to only a handful of genomes out of possibly many). The Findere sliding-window (`process_chunk`, the `win_min`/deque loop) would need reworking to run per `(sequence, genome)` over its sparse, sorted `(position, count)` list — detect runs of ≥`z` consecutive positions, window-min within each run — instead of today's dense `O(total_kmers × n_genomes)` scan. This is also a genuine complexity win (`O(hits log hits)` per genome vs. dense scan), not just memory.
+**Not covered by this sparsification**: `--detail`'s `cov` accumulator (`query.rs:304-308`) has the identical `n_genomes`-dense scaling problem and wasn't folded into points above. It doesn't need to be retained densely throughout processing, though — only the final JSON serialization (`emit_batch`) requires a dense `[u32]` per `(seq, genome)`, and only for the sequences actually being output with `--detail`. Densification can stay a late, output-time-only step, reconstructed from the sparse per-genome lists.
+
+**Secondary patterns available from `scatter.rs`/`superkmer.rs`, not yet in `query.rs`:**
+- `throttle()` + `CommonArgs::effective_max_open()` to bound concurrently-open input files (query.rs defines its own `QueryArgs`, doesn't reuse this).
+- Progress bar with EMA throughput + live active-worker gauges (`obisys::spinner`, `flat_active`/`transform_active` counters) — diagnostic value for locating the bottleneck.
+- `obisys::Reporter`/`Stage::start`/`stop` timing per phase (used by `index`, `filter`; absent from `query`).
+
+None of this is implemented yet — parked here as a coherent roadmap while the design is discussed further. Suggested dependency order: (1) I/O parallelism → (3) genome-aware chunk sizing → (4)–(9) staged/k-mer-deduped/NUMA-aware-partition-and-column-major/sparse query engine (larger refactor, biggest structural payoff — reuses `PartitionRunner` rather than inventing a new parallelism mechanism) → (2) parallel gzip (separate, orthogonal, tracked in chunkreader.md) → secondary diagnostics patterns.
+
+---
+
+## Implementation plan
+
+Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current `query` output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning `obikmer`, `obikpartitionner`, and `obicompactvec` — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense `KmerResults`) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if `rapidgzip-rs` is validated (see [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen)).
+
+Instrumentation is deliberately sequenced *before* the I/O fix (reordering the roadmap's own listed order), because every later phase's justification rests on a measurement ("to be measured, not assumed" appears throughout the roadmap above) — without it, phases 3–5 would be undertaken on faith.
+
+Performance measurement on the reference 192-core/8-NUMA machine is done by the project owner, not from this development environment (macOS, 16 cores — `PartitionRunner`'s NUMA pinning is Linux-only, so even phase 4's mechanism can't be functionally exercised for its actual purpose here). Each phase below is therefore written to be *self-measuring*: the debug-level logging it adds must be enough, on its own, to judge whether that phase's algorithmic choice paid off from a cluster run's logs, without needing to attach a profiler.
+
+### Conventions applied to every phase below
+
+**Debug logging.** Every phase that changes an algorithmic choice (not phase 0, which *is* the logging) adds `tracing::debug!`/`trace!` at points that let a cluster run's logs answer "did this help": counts, ratios, and timings that quantify the specific claim that phase makes — e.g. phase 3 must log how many MPHF `find` calls were saved by k-mer-level dedup (the whole justification for that phase), phase 4 must log per-column scan timings, phase 5 must log actual retained-memory / sparsity ratios achieved. Prefer one structured `debug!` per chunk (fields, not prose) over free-text — the cluster logs will be the only evidence available for judging these choices, so they need to be grep/awk-able, not just readable.
+
+**Unit tests.** This project's convention (`obiread`, `obikseq`, `obidebruinj`, `obicompactvec`, `obilayeredmap`, `obiskio`, `obifastwrite`) is `#[cfg(test)] #[path = "tests/<name>.rs"] mod tests;` at the bottom of the source file, with the actual test code in a sibling `src/tests/<name>.rs`. Neither `obikmer` nor `obikpartitionner` (the two crates phases 3 and 5 touch most) currently have a `src/tests/` directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.
+
+**Workflow (`jj`).** Work happens in a fresh `jj` commit, easy to abandon. `jj new` between phases is reasonable where it helps isolate a phase for review, but only when the working copy compiles at that point (project convention) — phase 3's internal sub-steps (batch dedup change, then `query_layer.rs` split, then the new return shape) will likely not each compile independently since they're one coupled change, so treat "commit boundary" and "plan phase boundary" as related but not forced to match 1:1; use judgement per phase rather than mechanically splitting on every bullet.
+
+### Phase 0 — Instrumentation (prerequisite for measuring every later phase)
+
+**Goal**: make core utilization, throughput, and per-stage timing visible on a real run, so phases 1–5 can be justified with numbers instead of assumption.
+
+- `obikmer/src/cmd/query.rs`: wrap `run()`'s main loop with `obisys::Reporter`/`Stage::start("query")`/`.stop()`, printed at the end via `rep.print()` — same pattern as `index.rs`/`filter.rs`.
+- Add an `obisys::spinner("query")` progress bar around the `pipe.apply(...)` loop, with an EMA throughput readout (bases/s or k-mers/s, mirroring `steps::scatter`'s `ema_rate` computation, `scatter.rs:88-118`) and live gauges for "chunks in flight" / "workers busy" — reuse the `AtomicU32` counter pattern from `scatter.rs` (`flat_active`, `transform_active`) rather than inventing a new one.
+- Add `max_open_files: Option<usize>` to `QueryArgs` and a `effective_max_open()` method mirroring `CommonArgs::effective_max_open()` (`obikmer/src/cli.rs:90-94`) — needed by phase 1's `throttle()` call. (`QueryArgs` can't just embed `CommonArgs` — it doesn't take `kmer_size`/`minimizer_size`/`partitions`/`level_max`/`theta` from the CLI, those come from the index metadata — so this is a small standalone addition, not a flatten.)
+- Add one structured `debug!` per `process_chunk` call: chunk byte size, sequence count, s-mer count, wall time, and (once later phases exist) the fields they add — this single log line is the baseline every later phase's own logging gets compared against.
+- **Validation**: none needed beyond "the numbers appear and look sane" — this phase changes no query logic or output.
+- **Deliverable used by every phase below**: a before/after throughput and core-utilization measurement on the reference machine.
+
+### Phase 1 — Parallel per-file I/O (fixes root cause of low core utilization)
+
+**Goal**: file opening, decompression, and chunk-boundary parsing run across the `n_workers` pool instead of serially in the pipe's dedicated source thread.
+
+- `obikmer/src/cmd/query.rs`:
+  - Replace the `paths.into_iter().flat_map(read_sequence_chunks_sized(...))` construction (current `run()`, building `all_chunks`) with `obipipeline::throttle(paths.into_iter(), args.effective_max_open())`, passed as the pipe's `input`.
+  - Add a new `QueryData::Path(PathBuf)` variant (alongside `Chunk`/`Output`) to carry the throttled path through the pipe's type-erasure mechanism.
+  - Add a new **first** pipe stage, `Flat`/fallible (`||?`), modeled on `scatter.rs:60-86` and `superkmer.rs:54-65`: given a `Throttled<PathBuf>`, call `read_sequence_chunks_sized(path, chunk_bytes)` and yield each `Rope` chunk, keeping `pw.guard` alive until the file's iterator is exhausted (reuse or adapt `scatter.rs`'s `GuardedIter` wrapper — same lifetime problem, same fix).
+  - The existing `process_chunk` transform stage becomes the pipe's **second** stage, unchanged in its own logic — it still receives one `Rope` chunk at a time, just no longer all coming from one serial source.
+  - `make_pipe!` invocation grows from one stage (`Chunk => Output`) to two (`Path => Chunk => Output`).
+- Log, per file: time spent waiting on the `throttle()` slot (queueing due to `max_open`), and time spent opening/decompressing/producing the first chunk — this is what directly proves (or disproves) that I/O is now spread across workers instead of serialized.
+- **Validation**: run `query` on a small multi-file input, diff output against the pre-change version — content must be identical; **record order across files is not guaranteed to be preserved** even before this change (chunk-level dispatch across `n_workers` already reorders completions), so the diff must be order-insensitive (sort by read id, or compare as sets) if it wasn't already.
+- **Measure**: core utilization on the reference machine with several large input files, compare against phase 0's baseline.
+
+### Phase 2 — Genome-aware chunk-size formula (fixes OOM)
+
+**Goal**: `chunk_bytes` reflects actual per-chunk memory (`O(n_genomes)`), not a fixed multiplier.
+
+- `obikmer/src/cmd/query.rs`, `run()`: `n_genomes` and `args.detail` are already computed above the `chunk_bytes` calculation (`n_genomes` at the top of `run()`, before line 407 in the current file) — reorder if needed, then replace:
+  ```rust
+  let computed = avail / (n_workers as u64 * 16);
+  ```
+  with a formula that scales the divisor by `n_genomes` (and roughly doubles it when `--detail` is set, since `cov` duplicates the per-genome accumulation): e.g. `per_chunk_multiplier = base_overhead + n_genomes as u64 * BYTES_PER_KMER_PER_GENOME * if detail { 2 } else { 1 }`, replacing the flat `16`. `BYTES_PER_KMER_PER_GENOME` should be derived from `KmerResults`'s actual layout (`4` bytes per `u32` entry in `data`, plus the `bool` in `in_index`, plus `win_min`'s equal-sized buffer) rather than guessed.
+  - `args.chunk_size` (manual `--chunk-size` override) keeps taking priority, unchanged.
+- Log the resolved `chunk_bytes`, `n_genomes`, and the estimated peak per-chunk memory (`chunk_bytes` × the same multiplier used to derive it) once at startup — lets a cluster run confirm the estimate was actually respected, not just that the process didn't get OOM-killed (which could also happen to be true for the wrong reason).
+- **Validation**: build a test index with a large `n_genomes` (e.g. hundreds), run `query` with default chunk sizing under a memory limit (`ulimit -v` or a cgroup), confirm it no longer gets OOM-killed and that memory scales as predicted when `n_genomes` grows.
+- **Note**: this phase is superseded once phase 5 lands (sparse retained memory no longer scales with `n_genomes × total_kmers` at all) — but it's needed immediately regardless, since phases 3–5 are a bigger, riskier change and users need a working `query` in the meantime.
+
+### Phase 3 — K-mer-level dereplication, staged MPHF/matrix lookup
+
+**Goal**: replace superkmer-level dedup with k-mer-level dedup (roadmap point 5), and split the fused MPHF-find/matrix-fetch (point 4) so stage 1's output is bucketed by layer and MPHF slot (point 6).
+
+- `obikmer/src/cmd/query.rs`:
+  - Replace `QueryBatch::from_records`'s dedup map (`HashMap<RoutableSuperKmer, Vec<SKDesc>>`, current `query.rs:112`) with a per-partition `HashMap<CanonicalKmer, Vec<(seq_idx: u32, pos: u32)>>`, built in the same `SuperKmerIter` pass: superkmer construction and partition routing (`part_idx` from the superkmer's minimizer hash) are unchanged, only the granularity of what gets deduplicated changes — each `CanonicalKmer` within a superkmer is inserted individually instead of the whole superkmer being the dedup key.
+  - **Verified**: `CanonicalKmer` (`obikseq/src/kmer.rs:390`, `pub type CanonicalKmer = CanonicalKmerOf<KLen>`) — the underlying `CanonicalKmerOf<L>` derives `Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash` (`kmer.rs:269`). Usable as a `HashMap`/`HashSet` key as-is, no change needed.
+- `obikpartitionner/src/query_layer.rs`:
+  - Split `QueryLayer::find_into` (`query_layer.rs:48-67`) into two methods: `find_slot(&self, kmer: CanonicalKmer) -> Option<usize>` (MPHF only, no matrix touch) and keep `fill_row` as-is for phase 4 to call later.
+  - Replace `query_partition_with`'s inner loop (`query_layer.rs:103-113`) with a version that, for each unique `CanonicalKmer`, calls `find_slot` across the partition's layers (stopping at first hit, same as today), and instead of immediately filling a row, records `(layer_idx, slot)`.
+  - New return shape for the partition-level query, replacing today's `on_hit(sk_idx, kmer_idx, row)` callback: `HashMap<layer_idx, HashMap<slot, Vec<(seq_idx, pos)>>>` (roadmap point 6) — built directly from the k-mer dedup map's `Vec<(seq_idx,pos)>` values, keyed by the resolved slot instead of the k-mer.
+- **This phase alone has no throughput benefit yet** (matrix fetch still happens, just deferred) beyond the k-mer-level dedup itself (fewer MPHF calls when queries have overlapping/repeated k-mers) — its purpose is to produce the input phase 4 needs. Land phase 3+4 together, not phase 3 alone, per the "don't split 3–5 across releases" note above.
+- Log, per chunk: total k-mer occurrences vs. unique `CanonicalKmer` count (the dedup ratio — the entire justification for this phase) and the resulting MPHF `find` call count. If the dedup ratio is close to `1.0` on real query data (little redundancy), that's the cluster run telling us this phase wasn't worth it — the logging needs to be able to say that, not just confirm the happy path.
+- **Unit tests**: create `obikmer/src/cmd/tests/query.rs` (new `src/tests/` dir for this crate, following the project's `#[cfg(test)] #[path = "tests/query.rs"] mod tests;` convention) and `obikpartitionner/src/tests/query_layer.rs` (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the `find_slot`/bucket-by-layer-and-slot construction against a small hand-built `QueryLayer` fixture, asserting the `(layer_idx, slot, seq_idx, pos)` tuples match what the old per-occurrence loop would have produced.
+
+### Phase 4 — Column-major matrix fetch (roadmap points 7–8) — implemented, NUMA parallelism deferred
+
+**Goal (revised during implementation)**: replace `fill_row`-per-hit (row-major, worst-case mmap locality) with a column-major scan. `PartitionRunner` turned out to be the wrong mechanism for this at this call granularity — see below; the column-major fetch itself is implemented and validated, without it.
+
+**What shipped:**
+- `obicompactvec`: the per-column accessors this phase needed **already existed** — `PersistentCompactIntMatrix::col_view(c)` and `PersistentBitMatrix::col_view(c)` are public, and `IntSliceView::get(slot)`/`BitSliceView::get(slot)` are public — the original plan underestimated how much of this plumbing the pairwise-distance code (`dump`/`select`/`stats`) had already required. The one real gap: `PersistentBitMatrix::col_view()` panics on the `Implicit` variant (the documented mono-genome fast path, `bitmatrix.rs`). Added `PersistentBitMatrix::get(c, slot) -> u32` (`bitmatrix.rs`), a non-panicking column-major point lookup that returns `1` for `Implicit` regardless of `c` — the smallest surface needed, not a new `col_get` API from scratch.
+- `obikpartitionner/src/query_layer.rs`: `query_partition_with` is now two explicit stages, matching roadmap points 6–8: **stage 1** (MPHF-only, per unique k-mer, bucket hits by `(layer_idx, slot)`, emits `QueryHit::Found`) then **stage 2** (per layer with ≥1 hit, column-major: for each genome column `g` in `0..layer.n_cols().min(n_genomes)`, scan that layer's bucketed slots and call `col_value(g, slot)`, emitting `QueryHit::Value(descs, g, value)` on nonzero). `QueryHit` is a single enum delivered through one `FnMut(QueryHit)` callback — an earlier two-closure design (`on_found` + `on_value`) didn't borrow-check, since the caller's single mutable accumulator (`KmerResults`) can't be captured by two separate `FnMut` closures passed to the same call.
+- `obikmer/src/cmd/query.rs`: `KmerResults::set` (row-major, whole-row-at-once) replaced by `mark_found` (stage 1: flag a position as indexed, independent of any genome's value) and `set_one` (stage 2: write one genome's value at one position). `QueryStats` extended with `n_columns_scanned`/`n_col_get_calls`, logged per chunk.
+- Total `get()`-equivalent calls are unchanged from the row-major version (`n_hits × n_cols` in the worst case, confirmed by `n_col_get_calls` in the debug log) — the win is locality (sequential access within one layer's column at a time, across `mmap`'d regions, instead of jumping across all columns per hit), exactly as predicted.
+
+**What did not ship, and why — `PartitionRunner` is architecturally the wrong tool here:**
+Reading `obikindex/src/numa.rs`'s actual `run()` body (not just its doc comments) shows every call spawns a timer thread **plus one OS thread per worker slot on every NUMA node** (`std::thread::scope` + one `s.spawn()` per node per `max_workers`) — on the 192-core/8-NUMA reference machine, that's on the order of 190+ fresh OS threads spawned **per call**. This is fine for its actual, established usage in this codebase (`merge.rs`, `index.rs`'s `build_layers`): one `PartitionRunner::new()` + one `run()` call per command invocation, amortised over a batch of ~256 long-running partitions. It is not fine for `query`'s call pattern: `query_partition_with` runs once per `(chunk, partition)`, potentially thousands of times per second — spawning ~190 OS threads that often to scan a handful of genome columns would very likely cost far more than the row-major approach it's meant to replace. This is exactly the "resolve empirically, don't assume" composition risk the roadmap flagged, just resolved by reading the mechanism's actual cost before wiring it in, rather than by measuring a regression on the cluster after the fact.
+The column-major loop in stage 2 is therefore a **plain sequential loop** for now — it captures the whole, provable locality win (roadmap point 8's actual claim) without adding any parallelism mechanism. Genome-column-level parallelism (point 8's "bonus" axis) and partition-level parallelism (point 7) are both deferred — not abandoned. Candidates for a follow-up, once there's a concrete profiling need: (a) `rayon`'s already-warm global pool (`into_par_iter()`) for the column axis specifically — cheap to invoke repeatedly since it doesn't spawn threads per call, though it's the same "naive rayon" pattern `numa_worker_pools.md` warns about for a *different* workload (random pointer-chasing over large hash maps); a column scan's access pattern (sequential reads within one `mmap`'d region) has a different contention profile and hasn't been shown to have the same problem — needs its own measurement, not an assumption either way; (b) restructuring so `PartitionRunner` is invoked once per whole `query` run (or per large batch of chunks) rather than per `(chunk, partition)`, amortising its spawn cost the way `merge`/`build_layers` do — a bigger structural change than this phase's scope.
+- Log (implemented): `QueryStats::n_columns_scanned`/`n_col_get_calls`, folded into the existing per-chunk `debug!("k-mer dedup + column-major fetch", ...)` line (`query.rs`) alongside phase 3's dedup counters.
+- **Unit tests**: extended `obikpartitionner/src/tests/query_layer.rs` (phase 3's file) — `query_partition_with`'s empty/missing-index paths updated for the new `QueryStats` fields and single-callback signature.
+- **Validation performed**: full workspace build + `cargo test --workspace`, zero failures. Functional validation against real indexes: (1) a single-genome index — output byte-identical to pre-phase-4 (same `kmer_count`/`kmer_strict_matches` on every record); (2) the existing 20-genome `benchmark/global_index_presence` index — runs correctly, `n_hits=0` for an unrelated query (expected: no shared k-mers between a plant read and a bacterial reference set), no panics, confirming the `Implicit`/multi-column bounds logic doesn't crash on a real multi-genome, mixed-format index; (3) **the critical correctness case**: built two single-sequence-pair test genomes, merged into one 2-genome index, queried with reads from both — reads from `genomeA` matched **only** `genomeA` (`kmer_count` identical to the pre-dedup occurrence count, zero leakage into `genomeB`'s column) and vice versa. This is the test that would have caught a column-index mixup, an off-by-one in `n_cols`, or cross-genome bleed from the stage-1/stage-2 split — it passed cleanly.
+- **Not yet done**: the microbenchmark comparing column-major vs. the old row-major access pattern's wall time / page-fault counters on a large-`n_genomes` layer — needs a realistically large multi-genome index and, for the page-fault counters specifically, Linux (not available from this development environment). Left for cluster validation alongside phases 1–3's own pending measurements.
+
+### Phase 5 — Sparse Findere rework (roadmap point 9)
+
+**Goal**: replace the dense `KmerResults`/`win_min` sliding-window scan with one operating on phase 4's sparse per-genome output.
+
+- `obikmer/src/cmd/query.rs`, `process_chunk`:
+  - Remove `KmerResults` (`query.rs:157-202`) and the dense `win_min` allocation (`query.rs:290-291`, sized `max_n_kmers × n_genomes`).
+  - Keep a lightweight dense `in_index: Vec<bool>` per chunk (sized `total_kmers`, independent of `n_genomes`) from phase 3's stage 1 — still needed for `kmer_missing` bookkeeping (leftmost-s-mer-of-window membership test), which phase 4's sparse structure doesn't carry (a k-mer with no genome hit has no entry there at all).
+  - New per-`(seq_idx, genome)` scan: for each genome's `Vec<(seq_idx, pos, count)>` (sorted, per phase 4), group by `seq_idx` (contiguous after sort), then within each sequence's positions detect runs of `pos, pos+1, pos+2, ...` of length ≥ `z`; within each run, the existing monotone-deque window-minimum logic (`query.rs`'s current `dq` loop, conceptually unchanged) applies — but the deque now only scans real entries in the run, never zero-filled gaps.
+  - Update `SeqAcc` accumulation and `emit_batch` to consume this per-genome sparse iteration instead of `results.val`/`results.is_in_index`.
+  - `--detail`/`cov`: build sparsely during the same scan (only positions with a confirmed contribution get an entry), densify into the `[u32]` JSON array only in `emit_batch`, only for genomes/sequences actually being serialized (per roadmap point 9's note, `query.rs:304-308`'s current dense allocation goes away).
+- Log, per chunk: total sparse entries retained vs. what the old dense `KmerResults` would have allocated (`total_smers × n_genomes`) — the sparsity ratio is this phase's entire reason for existing, so it must be directly visible in the logs, not inferred from process RSS. Also log the run-detection stats (number of runs found, average run length) — a low average run length relative to `z` would mean most positions still fail to form a full window, worth knowing.
+- **Unit tests**: `obikmer/src/cmd/tests/query.rs` (extended from phase 3) — the property test described below is the primary deliverable here, not an afterthought; write it as an actual `#[test]` (or a small internal fuzz/property-style loop over randomized fixtures if a property-testing crate isn't already a dependency — check before adding one, per this project's dependency-approval rule) rather than a one-off manual comparison.
+- **Validation — this is the correctness-critical phase**: property-test comparing old (dense, pre-phase-3) and new (sparse) implementations on the same randomized input/index fixtures, asserting identical `kmer_count`, `kmer_missing`, `kmer_strict_matches`, and (with `--detail`) `coverage` for every sequence. Keep both implementations compiled side by side (behind a debug-only flag or a temporary parallel code path) only for the duration of this validation; delete the dense path once parity is confirmed — per this project's own convention, superseded code is not kept "just in case."
+- **Update `docmd/architecture/query.md` itself**: once this phase lands, the "Findere z-window filter" section (which currently — correctly — describes the dense deque-over-`0..n_smers` scan) needs another pass to describe the sparse run-detection algorithm instead, as already flagged when this phase was discussed.
+
+**Implemented as planned, no deviations discovered this time.** What shipped:
+- `KmerResults` removed entirely, replaced by `SmerIndex` (`in_index: Vec<bool>` + `offsets`, unchanged size/purpose, renamed since it's no longer "results" — just the O(1)-per-position "was this k-mer found at all" bookkeeping) and `by_genome: Vec<Vec<(seq_idx, pos, value)>>` (one empty `Vec` per genome until a hit arrives — genomes with zero hits in a chunk cost nothing beyond the outer `Vec`'s own allocation).
+- New `sparse_findere_for_genome(hits, z, presence, threshold) -> (Vec<ConfirmedHit>, n_runs, total_run_len)` (`query.rs`): sorts one genome's raw hits by `(seq_idx, pos)`, detects maximal runs of consecutive `pos` within one sequence, runs the same monotone-deque window-minimum as before but scoped to each run (run-relative indices for eviction, absolute `pos` for computing `pos_out`). Presence/count adjustment (`u32::from(win_min >= threshold)` vs. raw) is applied inside this function, once per confirmed hit, rather than later during accumulation.
+- `process_chunk` restructured into three passes after the partition loop: (1) run `sparse_findere_for_genome` per genome, collecting `confirmed_by_genome` and run-detection stats; (2) accumulate `genome_totals` directly from `confirmed_by_genome` and mark a `confirmed_any: Vec<bool>` (sized `total_kmers_out`, not `× n_genomes`); (3) a position-only pass (`O(total_kmers_out)`, no genome factor) computing `kmer_count`/`kmer_missing` from `confirmed_any` + `SmerIndex`. `cov` (`--detail`) is populated by re-scanning `confirmed_by_genome` — only when `--detail` is actually set, otherwise skipped entirely.
+- Debug log added (`"sparse Findere"`): `n_dense_would_be` (`n_occurrences × n_genomes` — what the deleted dense path would have allocated), `n_sparse_entries` (what's actually retained), `n_runs`/`avg_run_len` (per the plan's ask, to see whether hits mostly fail to form complete windows).
+- **Unit tests**: `sparse_findere_matches_dense_reference_on_random_inputs` (`obikmer/src/cmd/tests/query.rs`) — 200 randomized cases (sequence count/length, `z`, presence/count mode, threshold, hit density from sparse to fully-dense) comparing `sparse_findere_for_genome` against `dense_reference_findere`, a faithful reimplementation of the deleted dense algorithm kept only as a test-local correctness oracle (no property-testing crate added — checked first, none was a workspace dependency; a small `std`-only xorshift64 PRNG stands in for one, deterministic and dependency-free). All 200 cases pass.
+- **Functional validation performed**: full workspace build + `cargo test --workspace`, zero failures. End-to-end against real indexes: baseline output (no flags) unchanged from pre-phase-5 recorded values on the same fixtures; `--count-missing` correct (`kmer_missing: 0` on a self-match); `--detail` correct — coverage array length matches `kmer_count`, and critically, re-ran the two-genome cross-contamination check from phase 4 with `--detail --count-missing`: `genomeA` reads show coverage sum `106` for `genomeA` and `0` for `genomeB` (and vice versa) — confirms the sparse-to-dense `cov` reconstruction doesn't leak across genomes either, not just the scalar `kmer_strict_matches` path.
+- This phase's roadmap item ("update the Findere z-window filter section") — done, see above; the "Algorithm" section's pseudocode was also updated, since it still named `KmerResults`/`SKDesc` from before phases 3–4.
+
+### Phase 6 — Parallel gzip decompression (independent, optional)
+
+Tracked separately in [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen); parked pending validation of `rapidgzip-rs` on real data. Not a dependency of, or a dependency for, phases 0–5 — `xopen` is shared infrastructure (`obiread`), phase 1 benefits from it but doesn't require it (phase 1 parallelises *across* files; this phase would additionally parallelise *within* one large file).
+
+### Cross-cutting risks
+
+- **Thread-budget oversubscription** (phase 4): the single biggest unresolved design question in this whole plan — see phase 4's composition note. Should be settled with real measurements early in phase 4, not assumed from the design alone.
+- **`obicompactvec` API surface growth** (phase 4): new public per-column accessors are additive (existing `fill_row`/`row` stay for other callers — `dump`, `select`, distance computations) — no breaking change expected, but worth checking `obicompactvec`'s other callers aren't already relying on `fill_row` being the only/cheapest access path in a way that would make maintaining two access patterns (row-major and column-major) a real maintenance cost rather than a one-off addition.
+- **`PersistentBitMatrix::Implicit`'s hardcoded `n_cols: 1` — resolved, not a bug.** `LayerMeta`'s own doc comment (`obicompactvec/src/layer_meta.rs:1-9`) states it is written "alongside `mphf.bin`" and read by `PersistentBitMatrix::open` "to determine `n_rows` for **the implicit (mono-genome presence/absence) case**" — i.e. `Implicit` is a documented single-genome fast path (no presence matrix needed when there is trivially one genome), not a generic "no matrix built yet" fallback. `n_cols: 1` is correct by design for the case it's meant to handle. Phase 4's column loop is safe as planned — this was worth checking once, doesn't need further action.

docmd/implementation/chunkreader.md

@@ -107,3 +107,19 @@ stateDiagram-v2
 `restart` is updated each time a `+` is found. When any state fails its expected input, the scan jumps back to `restart` and continues from there — guaranteeing that a `@` in a quality line cannot be accepted as a record start, because the `\n+\n` structure immediately following it (going backward) will not be found.
 
 Returns the byte offset of the `@` that starts the last complete record.
+
+---
+
+## Future work — parallel gzip decompression in `xopen`
+
+`obiread::xopen` (`xopen.rs`) decompresses gzip via `niffler` → `flate2`, which is single-threaded (standard DEFLATE has no parallel-decodable structure). For large local gzip inputs this single-threaded decompression can become the throughput bottleneck feeding the `query`/`index`/`superkmer` pipelines, since chunk/page production for a given file is serialized ahead of the worker pool.
+
+Candidate: special-case local, on-disk, gzip-magic-detected paths in `open_raw`/`xopen` to use [`rapidgzip-rs`](https://github.com/alekseizarubin/rapidgzip-rs) (`ReaderBuilder::new().parallelism(n).open(path)`, implements `Read + Seek`) instead of `niffler`, keeping `niffler` for every other case: `stdin` (`-`), HTTP(S) sources, and all non-gzip formats (bzip2, xz, zstd — less used in practice here).
+
+Constraints identified so far (not yet validated against real data):
+- Branch point must move earlier than the current `decompress()` call in `open_raw` — rapidgzip's fast path needs the file **path**, not an already-opened generic `Read`, so the gzip/local-file detection has to happen before the generic `File::open` + `niffler::send::get_reader` path is taken.
+- `stdin` and HTTP sources are not seekable — they stay on `niffler` regardless; the gain only applies to local on-disk `.gz` files.
+- `rapidgzip-sys` vendors a native C++ engine: requires CMake ≥ 3.17, a C++17 compiler, and `nasm` on x86 targets — a real build-toolchain addition, not just a pure-Rust crate.
+- Low maturity of the Rust binding at review time (2 GitHub stars, ~15 commits, April 2026 latest release) — the underlying C++ engine is validated (HPDC 2023 paper), but the binding itself has limited production track record.
+
+Decision: parked for now. Before adopting, validate on real data: throughput vs. `niffler` on representative large `.gz` inputs, and byte-for-byte correctness of decompressed output.

docmd/implementation/filtering.md

@@ -29,16 +29,23 @@ Multiple values separated by `|` are always OR-ed within the predicate.
 
 ### Path matching (`~` and `!~`)
 
-Metadata values can represent hierarchical taxonomic paths such as
+Metadata values can represent hierarchical concept paths such as
 `/Eukaryota/Viridiplantae/Streptophyta/Betulaceae/Betula/nana`.
 
-- **Absolute pattern** (starts with `/`): the value must start with the pattern
-  at a segment boundary.
-  `taxon~/Betulaceae/Betula` matches `/Betulaceae/Betula/nana` and
-  `/Betulaceae/Betula` but not `/Betulaceae/Betuloides/…`.
-- **Bare segment** (no leading `/`): the value must contain the pattern as an
-  exact path component anywhere.
-  `taxon~Betula` matches any path that has `Betula` as one of its segments.
+Stored taxonomy values always start with `/` (the root of the path).
+Query patterns do **not** need to start with `/` — a leading `/` is an optional
+start anchor, not a requirement.
+
+| Pattern form | Semantics |
+|---|---|
+| `A/B` | contiguous sub-path A then B, anywhere in the value |
+| `/A/B` | value starts with A then B |
+| `A/B$` | value ends with A then B |
+| `/A/B$` | value is exactly A then B |
+| `A@x/B` | A with class `x` followed by B with any class |
+
+- `taxon~/Betulaceae/Betula` matches any path that starts with `Betulaceae` then `Betula`.
+- `taxon~Betula` matches any path containing `Betula` as a segment, anywhere.
 
 ### Missing metadata key → NA
 

docmd/implementation/obicompactvec.md

@@ -5,12 +5,11 @@
 ```
 src/obicompactvec/src/
   lib.rs            public re-exports
-  traits.rs         BitSlice, BitSliceMut, IntSlice, IntSliceMut + conversion traits
+  views.rs          BitSliceView<'a>, IntSliceView<'a> — zero-copy read views
+  traits.rs         ColumnWeights, CountPartials, BitPartials (matrix aggregation)
   bitvec.rs         PersistentBitVec, PersistentBitVecBuilder, BitIter
-  memoryvec.rs      MemoryBitVec
   reader.rs         PersistentCompactIntVec (read-only)
   builder.rs        PersistentCompactIntVecBuilder (read-write)
-  memoryintvec.rs   MemoryIntVec
   tempintvec.rs     TempCompactIntVec, TempCompactIntVecBuilder (temp-file-backed)
   tempbitvec.rs     TempBitVec, TempBitVecBuilder (temp-file-backed)
   bitmatrix.rs      PersistentBitMatrix, PersistentBitMatrixBuilder
@@ -23,20 +22,20 @@ src/obicompactvec/src/
 
 ```mermaid
 graph TD
-    traits --> memoryvec
-    traits --> memoryintvec
-    bitvec --> memoryvec
-    bitvec --> bitmatrix
-    bitvec --> tempbitvec
+    views --> bitvec
+    views --> builder
+    views --> tempbitvec
+    views --> tempintvec
+    views --> bitmatrix
+    views --> intmatrix
     format --> reader
     format --> builder
     reader --> intmatrix
     reader --> tempintvec
     builder --> intmatrix
-    builder --> memoryintvec
     builder --> tempintvec
-    memoryvec --> traits
-    memoryintvec --> traits
+    bitvec --> tempbitvec
+    bitvec --> bitmatrix
     tempintvec --> intmatrix
     tempintvec --> bitmatrix
     tempbitvec --> intmatrix
@@ -62,7 +61,7 @@ All integer vectors use the same two-tier encoding regardless of storage backend
 
 **Overflow store** — maps slot index to a `u32` value ≥ 255:
 
-- In `MemoryIntVec` and `PersistentCompactIntVecBuilder`: a `HashMap<usize, u32>` in RAM.
+- In `PersistentCompactIntVecBuilder`: a `HashMap<usize, u32>` in RAM.
 - In `PersistentCompactIntVec` (reader): a sorted `[(slot: u64, value: u32)]` array in the mmap, with a sparse L1-resident index for binary search.
 
 ```mermaid
@@ -70,13 +69,12 @@ flowchart LR
     slot --> P["primary[slot]: u8"]
     P -->|"< 255"| V["value = byte (0–254)"]
     P -->|"= 255 sentinel"| OV["overflow store"]
-    OV -->|"MemoryIntVec / Builder"| HM["HashMap&lt;usize, u32&gt;\nin RAM"]
+    OV -->|"Builder"| HM["HashMap&lt;usize, u32&gt;\nin RAM"]
     OV -->|"PersistentCompactIntVec"| SA["sorted [(slot,value)] in mmap\n+ sparse L1 index"]
 ```
 
 **Key property — sentinel 255 = +∞ on `u8`:**
 
-This is exploited throughout the binary operations. On a `u8` comparison, 255 behaves as positive infinity:
 - `min(a, 255) = a` for all `a ≤ 254` → correct when only one side is overflow
 - `max(a, 255) = 255` → correct sentinel when either side is overflow
 - Only the **both-overflow** case requires reading actual values from the overflow store.
@@ -85,274 +83,60 @@ In practice, k (overflow count) ≪ n (total slots). Observed genomic data: ~0.0
 
 ---
 
-## Trait hierarchy
+## View types
 
-```mermaid
-classDiagram
-    class BitSlice {
-        <<trait>>
-        +len() usize
-        +words() &[u64]
-        +get(slot) bool
-        +count_ones() u64
-        +count_zeros() u64
-        +partial_jaccard_dist(other) (u64,u64)
-        +jaccard_dist(other) f64
-        +hamming_dist(other) u64
-    }
-    class BitSliceMut {
-        <<trait>>
-        +words_mut() &mut [u64]
-        +set(slot, value)
-        +copy_from(src)
-        +and(other)
-        +or(other)
-        +xor(other)
-        +not()
-    }
-    class IntSlice {
-        <<trait>>
-        +len() usize
-        +get(slot) u32
-        +primary_bytes() &[u8]
-        +overflow_entries() Iterator
-        +iter() Iterator
-        +sum() u64
-        +count_nonzero() u64
-        +cmp_scalar(pred) MemoryBitVec
-        +lt/leq/gt/geq(t) MemoryBitVec
-    }
-    class IntSliceMut {
-        <<trait>>
-        +set(slot, value)
-        +primary_bytes_mut() &mut [u8]
-        +clear_overflow()
-        +inc/dec/add_at(slot)
-        +copy_from(src)
-        +min/max/add/diff(other)
-        +count_bits(bits)
-    }
-    class IntToBit {
-        <<trait blanket>>
-        +to_bitvec(threshold) MemoryBitVec
-        +to_presence() MemoryBitVec
-    }
-    class BitToInt {
-        <<trait blanket>>
-        +to_intvec() MemoryIntVec
-    }
-    BitSliceMut --|> BitSlice : extends
-    IntSliceMut --|> IntSlice : extends
-    IntToBit --|> IntSlice : blanket T:IntSlice
-    BitToInt --|> BitSlice : blanket T:BitSlice
+The previous trait hierarchy (`BitSlice`, `BitSliceMut`, `IntSlice`, `IntSliceMut`) has been replaced by two concrete zero-copy view structs with inherent methods. Views are **`Copy`** — passing them is free. All read operations live on these two types.
+
+### `BitSliceView<'a>`
+
+```rust
+#[derive(Clone, Copy)]
+pub struct BitSliceView<'a> { pub(crate) words: &'a [u64], pub(crate) n: usize }
 ```
 
-### BitSlice (read-only)
+Bit `i` is at `words[i >> 6]` bit `i & 63` (LSB-first). Padding bits in the last word are zero.
 
-Required: `len()`, `words() -> &[u64]`.
+| Method | Cost |
+|---|---|
+| `len()`, `is_empty()` | O(1) |
+| `get(slot)` | O(1) |
+| `count_ones()` | POPCNT per word, O(n/64) |
+| `count_zeros()` | `n − count_ones()`, O(n/64) |
+| `iter() -> BitSliceIter<'a>` | O(1) setup, O(n) iteration |
+| `partial_jaccard_dist(other: BitSliceView)` | `(a&b).popcount`, `(a\|b).popcount` per word, O(n/64) |
+| `jaccard_dist(other: BitSliceView)` | from partial, O(n/64) |
+| `hamming_dist(other: BitSliceView)` | `(a^b).popcount` per word, O(n/64) |
 
-Bit `i` is at `words()[i >> 6]` bit `i & 63` (LSB-first). Padding bits in the last word are always zero — this invariant must be maintained by all implementors.
+`BitSliceIter<'a>`: word-level scan; one word per 64 iterations.
 
-| Provided method | Implementation | Cost |
-|---|---|---|
-| `is_empty()` | `len() == 0` | O(1) |
-| `get(slot)` | word extract | O(1) |
-| `count_ones()` | POPCNT per word | O(n/64) |
-| `count_zeros()` | `n − count_ones()` | O(n/64) |
-| `partial_jaccard_dist(other)` | `(a&b).popcount`, `(a\|b).popcount` per word | O(n/64) |
-| `jaccard_dist(other)` | from partial | O(n/64) |
-| `hamming_dist(other)` | `(a^b).popcount` per word | O(n/64) |
+### `IntSliceView<'a>`
 
-### BitSliceMut: BitSlice (mutable)
-
-Required: `words_mut() -> &mut [u64]`.
-
-All bulk operations work at the word level (64 bits/iteration). The compiler auto-vectorizes these loops to AVX2/AVX-512. The zero-padding invariant is maintained: `not()` re-masks the last word after flipping.
-
-| Provided method | Implementation | Cost |
-|---|---|---|
-| `set(slot, value)` | OR / AND-NOT on one word | O(1) |
-| `copy_from(src)` | `copy_from_slice` = memcpy | O(n/64) |
-| `and(other)` | `w &= o` per word | O(n/64) |
-| `or(other)` | `w \|= o` per word | O(n/64) |
-| `xor(other)` | `w ^= o` per word | O(n/64) |
-| `not()` | `w ^= u64::MAX` per word, then mask last | O(n/64) |
-
-**No overflow complexity here.** The packed `u64` representation is already the natural unit for SIMD operations. No sentinel, no HashMap — just bitwise word ops.
-
----
-
-### IntSlice (read-only)
-
-Required:
-- `len() -> usize`
-- `get(slot) -> u32` — handles sentinel transparently (binary search into overflow for persistent, HashMap for memory)
-- `primary_bytes() -> &[u8]` — raw primary array including 255 sentinels
-- `overflow_entries() -> impl Iterator<Item = (usize, u32)>` — (slot, true_value) pairs for all overflow slots
-
-| Provided method | Default implementation | Note |
-|---|---|---|
-| `is_empty()` | `len() == 0` | |
-| `iter()` | `(0..n).map(\|i\| self.get(i))` | Overridden in all concrete types |
-| `sum()` | `iter().map(\|v\| v as u64).sum()` | Overridden in concrete types |
-| `count_nonzero()` | `iter().filter(\|v\| *v > 0).count()` | Overridden in concrete types |
-| `lt(t)` | `cmp_scalar(\|v\| v < t)` | |
-| `leq(t)` | `cmp_scalar(\|v\| v <= t)` | |
-| `gt(t)` | `cmp_scalar(\|v\| v > t)` | |
-| `geq(t)` | `cmp_scalar(\|v\| v >= t)` | |
-| `cmp_scalar(pred)` | two-pass (see below) | |
-
-**`cmp_scalar` algorithm — two passes:**
-
-```
-Pass 1 — byte scan, O(n):
-  for s in 0..n:
-    b = primary[s]
-    if b < 255 AND pred(b as u32):
-      set bit s in result word
-
-Pass 2 — overflow fixup, O(k):
-  for (s, val) in overflow_entries():
-    if pred(val): set bit s in result word
+```rust
+#[derive(Clone, Copy)]
+pub struct IntSliceView<'a> {
+    pub(crate) primary:      &'a [u8],
+    pub(crate) overflow_raw: &'a [u8],   // sorted [(slot:u64, value:u32)] entries
+    pub(crate) n_overflow:   usize,
+    pub(crate) n:            usize,
+}
 ```
 
-Pass 1 reads only the primary byte array — no HashMap access. For simple predicates (`geq`, `lt`, etc.) the compiler inlines `pred` and can auto-vectorize the byte comparison loop. Pass 2 handles the O(k) overflow slots that were left as 0 in pass 1.
+`overflow_raw` contains `n_overflow` entries of `OVERFLOW_ENTRY_SIZE` bytes each, sorted by slot. The sort invariant is established at `close()`/`freeze()` time.
 
-Previous implementation: `pred(self.get(s))` for every slot → O(n log k) due to binary search in overflow. New: O(n) + O(k).
+| Method | Cost |
+|---|---|
+| `len()`, `is_empty()` | O(1) |
+| `primary_bytes()` | O(1) |
+| `overflow_entries() -> impl Iterator<(usize,u32)>` | O(n_overflow) iteration |
+| `get(slot)` | O(1) primary; binary search O(log k) for overflow slots |
+| `iter() -> IntSliceViewIter<'a>` | merge scan, O(n + k) |
+| `sum()` | byte scan + overflow, O(n + k) |
+| `count_nonzero()` | byte scan, O(n) |
+| Distance methods (`bray_dist`, `euclidean_dist`, `jaccard_dist`, …) | O(n + k) |
 
----
+`IntSliceViewIter<'a>`: merge scan using `overflow_pos` index. Requires sorted overflow — guaranteed by the construction lifecycle.
 
-### IntSliceMut: IntSlice (mutable)
-
-Required:
-- `set(slot, value: u32)` — writes primary byte (or 255 + overflow entry if value ≥ 255); removes stale overflow entry if value drops below 255
-- `primary_bytes_mut() -> &mut [u8]` — direct mutable access to the primary array
-- `clear_overflow()` — empties the entire overflow store
-
-The required methods expose the encoding internals. All provided methods are implemented in terms of these three + the `IntSlice` required methods.
-
-| Provided method | Hot path | Overflow case | Cost |
-|---|---|---|---|
-| `inc(slot)` | `get` + `set` | — | O(1) or O(log k) |
-| `dec(slot)` | `get` + `set` (saturating) | — | O(1) or O(log k) |
-| `add_at(slot, delta)` | `get` + `set` (saturating) | — | O(1) or O(log k) |
-| `copy_from(src)` | `copy_from_slice` + `clear_overflow` + replay overflows | — | O(n) + O(k) |
-| `min(other)` | byte-level min, O(n) | both-overflow fixup, O(k) | O(n) |
-| `max(other)` | byte-level max, O(n) | pre-pass on other's overflows, O(k) | O(n) |
-| `add(other)` | byte add when both < 255, O(n) | `get` + `+` when either = 255 | O(n) |
-| `diff(other)` | byte saturating_sub when self < 255, O(n) | `get` + `saturating_sub` when self = 255 | O(n) |
-| `count_bits(bits)` | iterate set bits via word scan | — | O(n_ones) |
-| `cmp_scalar` | inherited from IntSlice | — | O(n) + O(k) |
-
-**`min` algorithm:**
-
-Exploits 255 = +∞: `u8::min(a, 255) = a` and `u8::min(255, b) = b`. Only the case where both sides are ≥ 255 needs actual overflow values.
-
-```mermaid
-flowchart TD
-    A["min(self, other)"] --> B["snapshot self_ov: Vec&lt;(slot,val)&gt;\nsnapshot other_ov: HashMap&lt;slot,val&gt;"]
-    B --> C["clear_overflow()"]
-    C --> D["Pass 1 — byte min, SIMD-vectorizable\nprimary[s] = min(self[s], other[s])  ∀s"]
-    D --> E["Pass 2 — both-overflow fixup\nfor (slot, self_val) in self_ov"]
-    E --> F{"slot ∈ other_ov?"}
-    F -->|yes| G["set(slot, min(self_val, other_ov[slot]))"]
-    F -->|no| H["byte pass wrote other.primary &lt; 255\nclear_overflow removed stale entry\nno action"]
-    G --> I[done]
-    H --> I
-```
-
-Overflow entries where only self was overflow are correctly handled: after `clear_overflow` + byte pass, `self.primary[slot] = min(255, other.primary[slot]) = other.primary[slot]` (which is < 255). No overflow entry — correct.
-
-**`max` algorithm:**
-
-Exploits 255 = +∞: `u8::max(a, 255) = 255` → any slot where either side is overflow will have sentinel 255 in the primary after the byte pass. The byte pass cannot distinguish "self had overflow and other did not" from "self was just written to 255 by the byte pass".
-
-Solution: read and update self's original value at other's overflow slots *before* the byte pass overwrites them.
-
-```mermaid
-flowchart TD
-    A["max(self, other)"] --> B["Pre-pass O(k_other)\nfor (slot, other_val) in other.overflow_entries()"]
-    B --> C["self_val = self.get(slot)\nself.set(slot, max(self_val, other_val))"]
-    C --> D["Pass 1 — byte max, SIMD-vectorizable\nprimary[s] = max(self[s], other[s])  ∀s"]
-    D --> E["Overflow slots: max(255,255)=255\nprimary unchanged\noverflow entry from pre-pass preserved"]
-    E --> F[done]
-```
-
-After the pre-pass, self.primary[slot] = 255 for all slots in other's overflow. The byte pass leaves those 255s intact. Self's own overflow slots not in other's overflow are also 255 in primary — byte max(255, b < 255) = 255, unchanged. Correct in all cases.
-
-**`add` algorithm:**
-
-No sentinel property useful for add: any pair (sb, ob) with sb + ob ≥ 255 creates a new overflow entry, even when neither input was overflow. Cannot simplify via byte arithmetic.
-
-```
-for s in 0..n:
-  sb = self.primary[s]
-  ob = other.primary[s]
-  if sb < 255 AND ob < 255:      // hot path: no HashMap
-    sum = sb as u32 + ob as u32
-    if sum < 255: self.primary[s] = sum as u8   // direct byte write
-    else:         self.set(s, sum)               // creates overflow if needed
-  else:                           // at least one is overflow
-    self.set(s, self.get(s) + other.get(s))
-```
-
-```mermaid
-flowchart TD
-    A["add(self, other)"] --> B{"sb &lt; 255\nAND ob &lt; 255"}
-    B -->|"yes — hot path\nno HashMap"| C{"sb + ob &lt; 255"}
-    C -->|yes| D["primary[s] = sum as u8\nsingle byte write"]
-    C -->|no| E["set(s, sum)\ncreates overflow entry"]
-    B -->|"no — ≥1 side is overflow"| F["self_val = self.get(s)\nother_val = other.get(s)\nset(s, self_val + other_val)"]
-    D --> Z[next slot]
-    E --> Z
-    F --> Z
-```
-
-The `+` on `u32` values is exact (no `saturating_add`). Overflow at u32 level panics in debug — not a real risk for kmer counts. The hot path (both < 255, sum < 255) is a single byte write with no HashMap access.
-
-**`diff` (saturating sub) algorithm:**
-
-`saturating_sub(a, b) = a − min(a, b) = max(0, a − b)`. Key insight: if self's primary byte < 255, the result is always < 255 (result ≤ a), so no new overflow entries are created and no overflow lookup is needed for self. Only self's overflow slots (primary = 255) need `get()`.
-
-| sb | ob | result | get() needed |
-|----|----|--------|-------------|
-| < 255 | < 255 | `sb.saturating_sub(ob)` < 255 | none |
-| < 255 | 255 | 0 (b ≥ 255 > a) | none |
-| 255 | < 255 | `self.get(s) − ob` | self only |
-| 255 | 255 | `self.get(s) − other.get(s)` | both |
-
-```mermaid
-flowchart TD
-    A["diff(self, other)"] --> B{"sb &lt; 255\nself not overflow"}
-    B -->|"yes — hot path O(n)"| C{"ob &lt; 255"}
-    C -->|yes| D["primary[s] = sb.saturating_sub(ob)\nbyte write, no HashMap"]
-    C -->|"no: b ≥ 255 > a"| E["primary[s] = 0"]
-    B -->|"no — cold path O(k_self)"| F["self_val = self.get(s)"]
-    F --> G{"ob &lt; 255"}
-    G -->|yes| H["other_val = ob as u32"]
-    G -->|no| I["other_val = other.get(s)"]
-    H --> J["set(s, self_val.saturating_sub(other_val))"]
-    I --> J
-    D --> Z[next slot]
-    E --> Z
-    J --> Z
-```
-
-Overflow entries that drop below 255 (case sb=255, result < 255) are removed by `set()`. Overflow entries that remain ≥ 255 are updated. Correct in all four cases.
-
-**`count_bits` algorithm:**
-
-Increments self at each slot where the corresponding bit in `bits` is set. Iterates `bits.words()` and skips zero words entirely — O(n_ones) rather than O(n).
-
-```
-for (w_idx, word) in bits.words():
-  if word == 0: continue
-  base = w_idx * 64
-  while word != 0:
-    bit = trailing_zeros(word)
-    self.inc(base + bit)
-    word &= word − 1        // clear lowest set bit
-```
+**Builder `view()` vs reader `view()`:** `PersistentCompactIntVecBuilder` stores overflow as an unsorted `HashMap`, not raw bytes. Its `view()` returns an `IntSliceView` with `overflow_raw = &[]` and `n_overflow = 0`. This is intentional — the view is primarily useful after `freeze()`. During building, callers that need overflow use `overflow_entries()` directly.
 
 ---
 
@@ -360,142 +144,149 @@ for (w_idx, word) in bits.words():
 
 ```mermaid
 classDiagram
-    class MemoryBitVec {
-        -words: Vec~u64~
-        -n: usize
-        +iter() BitIter
-        +ones(n) Self
-        +persist(path) Builder
+    class BitSliceView {
+        +words: &[u64]
+        +n: usize
+        +get(slot) bool
+        +count_ones() u64
+        +iter() BitSliceIter
+        +jaccard_dist/hamming_dist(other: BitSliceView)
     }
-    class MemoryIntVec {
-        -primary: Vec~u8~
-        -overflow: HashMap~usize,u32~
-        -n: usize
-        +iter() MemoryIntIter
-        +filled(n, value) Self
-        +persist(path) Builder
+    class IntSliceView {
+        +primary: &[u8]
+        +overflow_raw: &[u8]
+        +n_overflow: usize
+        +n: usize
+        +get(slot) u32
+        +iter() IntSliceViewIter
+        +overflow_entries() Iterator
+        +bray_dist/euclidean_dist/…(other: IntSliceView)
     }
     class PersistentBitVec {
         -mmap: Mmap
         -n: usize
+        +view() BitSliceView
+        +get(slot) bool
+        +count_ones/zeros() u64
         +iter() BitIter
-        +count_ones() u64
+        +partial_jaccard_dist(&Self) (u64,u64)
+        +jaccard_dist/hamming_dist(&Self) …
     }
     class PersistentBitVecBuilder {
         -mmap: MmapMut
         -n: usize
-        +close()
-        +build_from(src, path)
-        +build_from_counts(src, t, path)
+        +view() BitSliceView
+        +set(slot, bool)
+        +or/and/xor/not(BitSliceView)
+        +copy_from(BitSliceView)
+        +close() / finish() → PersistentBitVec
     }
     class PersistentCompactIntVec {
         -mmap: Mmap
-        -n usize
-        -n_overflow usize
-        -step usize
+        -n: usize
+        -n_overflow: usize
+        -step: usize
         -index: Vec~(usize,usize)~
-        +iter() Iter
+        +view() IntSliceView
         +get(slot) u32
-        +sum() u64
+        +iter() Iter
+        +sum/count_nonzero() u64
+        +bray_dist/euclidean_dist/… (&Self)
     }
     class PersistentCompactIntVecBuilder {
         -mmap: MmapMut
         -n: usize
         -overflow: HashMap~usize,u32~
-        +set(slot, value)
-        +close()
-        +build_from(src, path)
+        +view() IntSliceView
+        +set(slot, u32) / get(slot) u32
+        +inc / inc_present / inc_present_fast
+        +inc_predicate / inc_predicate_fast
+        +add/min/max/diff/mask_with(…View)
+        +primary_bytes/primary_bytes_mut()
+        +close() / finish() → PersistentCompactIntVec
     }
 
-    MemoryBitVec ..|> BitSlice
-    MemoryBitVec ..|> BitSliceMut
-    PersistentBitVec ..|> BitSlice
-    PersistentBitVecBuilder ..|> BitSlice
-    PersistentBitVecBuilder ..|> BitSliceMut
-    MemoryIntVec ..|> IntSlice
-    MemoryIntVec ..|> IntSliceMut
-    PersistentCompactIntVec ..|> IntSlice
-    PersistentCompactIntVecBuilder ..|> IntSlice
-    PersistentCompactIntVecBuilder ..|> IntSliceMut
-
+    PersistentBitVec --> BitSliceView : view()
+    PersistentBitVecBuilder --> BitSliceView : view()
+    PersistentCompactIntVec --> IntSliceView : view()
+    PersistentCompactIntVecBuilder --> IntSliceView : view() (primary only)
     PersistentBitVecBuilder --> PersistentBitVec : close() then open()
     PersistentCompactIntVecBuilder --> PersistentCompactIntVec : close() then open()
 ```
 
-### Memory types
+### `PersistentBitVec` / `PersistentBitVecBuilder`
 
-**`MemoryBitVec`**
+`PersistentBitVec` is the read-only type. `view()` returns a `BitSliceView<'_>` over the mmap word array. Direct inherent methods delegate to the view: `count_ones()`, `count_zeros()`, `partial_jaccard_dist(&Self)`, `jaccard_dist(&Self)`, `hamming_dist(&Self)`.
 
-```rust
-struct MemoryBitVec { words: Vec<u64>, n: usize }
-```
-
-Implements `BitSlice` + `BitSliceMut`. Owns its word array. Used as the result type of `cmp_scalar` / filter operations and as an intermediate for bit-level computations.
-
-Std ops: `BitAnd`, `BitOr`, `BitXor`, `Not` (owned and borrowed), `BitAndAssign`, `BitOrAssign`, `BitXorAssign` — all delegate to `BitSliceMut` methods.
-
-`iter()` returns a `BitIter<'_>` (word-level, see below).
-
-**`MemoryIntVec`**
-
-```rust
-struct MemoryIntVec {
-    primary:  Vec<u8>,
-    overflow: HashMap<usize, u32>,
-    n:        usize,
-}
-```
-
-Implements `IntSlice` + `IntSliceMut`. Overrides: `iter()` → inherent `iter()` (merge-scan), `sum()`, `count_nonzero()`.
-
-`IntSlice` required impls: `primary_bytes()` → `&self.primary`; `overflow_entries()` → `self.overflow.iter().map(...)`.
-
-`IntSliceMut` required impls: `set()` writes to `self.primary[slot]` and inserts/removes from `self.overflow`; `primary_bytes_mut()` → `&mut self.primary`; `clear_overflow()` → `self.overflow.clear()`.
-
-Std ops: `Add<&B>`, `Sub<&B>` (owned and borrowed), `AddAssign<&B>`, `SubAssign<&B>` — delegate to `IntSliceMut::add` / `diff`.
-
-`From<&S: IntSlice>`: copies primary bytes + overflow entries. O(n) + O(k).
-
----
-
-### Persistent types
-
-**`PersistentBitVec` / `PersistentBitVecBuilder`**
-
-See `persistent_bit_vec.md`. `PersistentBitVec` is read-only (implements `BitSlice`). `PersistentBitVecBuilder` is read-write (implements `BitSlice` + `BitSliceMut`).
-
-`BitIter<'a>` — shared iterator type for both `MemoryBitVec` and `PersistentBitVec`:
+`BitIter<'a>` — exported iterator for `PersistentBitVec::iter()`:
 
 ```rust
 pub struct BitIter<'a> { pub(crate) words: &'a [u64], pub(crate) slot: usize, pub(crate) n: usize }
 ```
 
-Word-level scan: `(words[slot >> 6] >> (slot & 63)) & 1 != 0`. One word serves 64 iterations. `pub type MemoryBitIter<'a> = BitIter<'a>` preserves the public API name.
+`PersistentBitVecBuilder` is the read-write type. Mutation operations accept `BitSliceView<'_>`:
 
-**`PersistentCompactIntVec` / `PersistentCompactIntVecBuilder`**
+| Method | Cost |
+|---|---|
+| `set(slot, bool)` | O(1) |
+| `view() -> BitSliceView<'_>` | O(1) |
+| `or/and/xor(BitSliceView)` | word-level, O(n/64), SIMD-friendly |
+| `not()` | `w ^= u64::MAX` per word, re-masks last word | O(n/64) |
+| `copy_from(BitSliceView)` | `copy_from_slice` | O(n/64) |
 
-See `persistent_compact_int_vec.md` for file format and lifecycle.
+### `PersistentCompactIntVec` / `PersistentCompactIntVecBuilder`
 
-`PersistentCompactIntVec` implements `IntSlice`. Overrides: `iter()` → inherent merge-scan `Iter`; `sum()`; `count_nonzero()`. `overflow_entries()` returns a sequential scan `(0..n_overflow).map(|i| (data_slot(i), data_value(i)))` — no binary search since entries are stored sorted.
+`PersistentCompactIntVec` is the read-only type. `view()` returns an `IntSliceView<'_>` over the mmap primary and overflow arrays. Inherent `iter()` is a merge scan (`Iter` struct). Inherent `sum()` and `count_nonzero()` use fast byte-scan helpers.
 
-`PersistentCompactIntVecBuilder` implements `IntSlice` + `IntSliceMut`. `iter()` is NOT overridden (default `get`-per-slot) because the overflow `HashMap` is unsorted. `sum()` and `count_nonzero()` are overridden using `byte_sum` / `byte_count_nonzero` on the mmap primary slice — avoids per-slot overhead.
+`PersistentCompactIntVecBuilder` is the read-write type. Mutation methods on the builder fall into two categories:
 
-**Override rationale:** the default `iter()`, `sum()`, `count_nonzero()` on `IntSlice` call `self.get(s)` per slot, which is O(log k) binary search for `PersistentCompactIntVec`. Overrides provide O(n + k) merge-scan or O(n) byte scan instead.
+**Point mutations:**
 
----
+| Method | Note |
+|---|---|
+| `set(slot, u32)` | writes primary[slot] or 255+overflow |
+| `get(slot) -> u32` | reads primary byte or HashMap |
+| `inc(slot)` | `get` + `set`, O(1) |
 
-### IntSlice implementors — override summary
+**Bulk computation methods** — accept view arguments:
 
-| Type | `iter()` | `sum()` | `count_nonzero()` |
-|------|----------|---------|-------------------|
-| `MemoryIntVec` | inherent merge-scan ✓ | `byte_sum` ✓ | `byte_count_nonzero` ✓ |
-| `PersistentCompactIntVecBuilder` | default (get-per-slot) | `byte_sum` on mmap ✓ | `byte_count_nonzero` on mmap ✓ |
-| `PersistentCompactIntVec` | inherent merge-scan Iter ✓ | inherent `sum()` ✓ | inherent `count_nonzero()` ✓ |
-| `TempCompactIntVec` | delegates to inner `PersistentCompactIntVec` | delegates | delegates |
-| `TempCompactIntVecBuilder` | default (get-per-slot) | delegates to builder | delegates to builder |
-| `PackedIntCol<'a>` | inherent PackedIntColIter ✓ | byte_sum ✓ | byte_count_nonzero ✓ |
+| Method | Semantics | Overflow |
+|---|---|---|
+| `inc_present(BitSliceView)` | `+= 1` at each 1-bit | via `inc`, safe for any group size |
+| `inc_present_fast(BitSliceView)` | same, raw u8 `+= 1` | `debug_assert` no 255 reached |
+| `inc_predicate(IntSliceView, pred)` | `+= 1` where `pred(col[s])` | two-pass, safe |
+| `inc_predicate_fast(IntSliceView, pred)` | same, raw u8 | `debug_assert` no 255 reached |
+| `add(IntSliceView)` | `self[s] += other[s]` | primary fast path + overflow fallback |
+| `min(IntSliceView)` | byte min + both-overflow fixup | see algorithm below |
+| `max(IntSliceView)` | pre-pass + byte max | see algorithm below |
+| `diff(IntSliceView)` | saturating sub | self<255 hot path |
+| `mask_with(BitSliceView)` | zeros slots where mask bit = 0 | O(n_zeros) |
 
-`PackedIntCol` is used internally by `PersistentCompactIntMatrix` (packed format) for column views.
+**`inc_present_fast` / `inc_predicate_fast` invariant:** caller guarantees no counter reaches 255 during the operation (group size < 255 for `inc_present_fast`, or chunk size < 255 for `inc_predicate_fast`). Violation is caught by `debug_assert` in dev builds.
+
+**`min` algorithm:**
+
+Exploits 255 = +∞: byte-level min is correct unless both sides are overflow.
+
+```
+snapshot self_ov: Vec<(slot,val)>
+snapshot other_ov: HashMap<slot,val>
+clear_overflow()
+Pass 1 — byte min, SIMD-vectorizable, O(n)
+Pass 2 — both-overflow fixup, O(k_self):
+  for (slot, self_val) in self_ov:
+    if slot ∈ other_ov: set(slot, min(self_val, other_ov[slot]))
+```
+
+**`max` algorithm:**
+
+Cannot do byte max first — `max(255, b<255)=255` overwrites self's original overflow value. Pre-pass reads self's value at other's overflow slots before the byte pass.
+
+```
+Pre-pass O(k_other): for (slot, other_val) in other.overflow_entries():
+  set(slot, max(self.get(slot), other_val))
+Pass 1 — byte max, SIMD-vectorizable, O(n)
+```
 
 ---
 
@@ -505,30 +296,22 @@ Four matrix types, two encodings × two formats:
 
 | | Columnar format | Packed format |
 |---|---|---|
-| **Bit** | `PersistentBitMatrix` | — |
-| **Int** | `PersistentCompactIntMatrix` (columnar) | `PersistentCompactIntMatrix` (packed) |
+| **Bit** | `PersistentBitMatrix` (Columnar variant) | `PersistentBitMatrix` (Packed variant) |
+| **Int** | `PersistentCompactIntMatrix` (Columnar variant) | `PersistentCompactIntMatrix` (Packed variant) |
 
-`PersistentCompactIntMatrix` is an enum behind a transparent API — the caller does not see whether the on-disk format is columnar (one `.pciv` per column) or packed (one `.pcmx` file interleaving all columns). `col(c)` and `col_slice(c)` return column views that implement `IntSlice`.
+Both matrix types are enums (`Columnar` / `Packed` / `Implicit` for bit) behind a transparent API. `col_view(c)` returns the appropriate view directly:
 
-`pack_compact_int_matrix` and `pack_bit_matrix` convert a columnar matrix to packed format.
+```rust
+// PersistentBitMatrix
+pub fn col_view(&self, c: usize) -> BitSliceView<'_>
 
-For details see `persistent_compact_int_vec.md` and `persistent_bit_vec.md`.
+// PersistentCompactIntMatrix
+pub fn col_view(&self, c: usize) -> IntSliceView<'_>
+```
 
----
+No wrapper enums (`BitColView`, `IntColView`): the caller receives a `Copy` view struct immediately usable with any view method or bulk builder method.
 
-## Conversion traits
-
-Four blanket-impl traits on top of `BitSlice` / `IntSlice`:
-
-**`IntToBit: IntSlice`**
-- `to_bitvec(threshold: u32) -> MemoryBitVec` — bit set iff value ≥ threshold (delegates to `geq`)
-- `to_presence() -> MemoryBitVec` — bit set iff value ≥ 1 (delegates to `geq(1)`)
-
-**`BitToInt: BitSlice`**
-- `to_intvec() -> MemoryIntVec` — expands each bit to a `u8` (0 or 1) in a new primary array
-- Uses a `static EXPAND_BYTE: [[u8; 8]; 256]` lookup table — 8 bits expanded per byte, word-level outer loop
-
-Both `IntToBit` and `BitToInt` are implemented for all `T: IntSlice` / `T: BitSlice` via blanket impls.
+`pack_compact_int_matrix` and `pack_bit_matrix` convert columnar → packed format.
 
 ---
 
@@ -549,37 +332,37 @@ trait ColumnWeights: Send + Sync {
 
 Abstract required methods: `partial_bray`, `partial_euclidean`, `partial_threshold_jaccard`, `partial_relfreq_bray`, `partial_relfreq_euclidean`, `partial_hellinger`.
 
-**Additivity rule:** self-contained partials (`partial_bray`, `partial_euclidean`, `partial_threshold_jaccard`) can be element-wise summed across all `(partition, layer)` pairs before applying the finalisation. Normalised partials (`partial_relfreq_*`, `partial_hellinger`) require the **global** `col_weights` (accumulated across all layers and all partitions) as parameter — not per-layer or per-partition weights.
+**Additivity rule:** self-contained partials (`partial_bray`, `partial_euclidean`, `partial_threshold_jaccard`) can be element-wise summed across all `(partition, layer)` pairs. Normalised partials (`partial_relfreq_*`, `partial_hellinger`) require the **global** `col_weights` (accumulated across all layers and all partitions) as parameter.
 
-**`partial_threshold_jaccard` returns `(inter, union)`**, not a single matrix, because `union[i,j]` depends on both columns simultaneously and cannot be reconstructed from per-column statistics.
+**`partial_threshold_jaccard` returns `(inter, union)`** because `union[i,j]` depends on both columns simultaneously.
 
-Provided finalisations (default implementations):
+Provided finalisations:
 
 | Finalisation | Formula |
 |---|---|
 | `bray_dist_matrix()` | `1 − 2·partial_bray[i,j] / (w[i] + w[j])` |
 | `euclidean_dist_matrix()` | `√partial_euclidean[i,j]` |
 | `threshold_jaccard_dist_matrix(t)` | `1 − inter[i,j] / union[i,j]` |
-| `relfreq_bray_dist_matrix()` | `1 − partial_relfreq_bray[i,j]` (two-pass: col_weights then partial) |
+| `relfreq_bray_dist_matrix()` | `1 − partial_relfreq_bray[i,j]` |
 | `relfreq_euclidean_dist_matrix()` | `√partial_relfreq_euclidean[i,j]` |
 | `hellinger_dist_matrix()` | `√partial_hellinger[i,j] / √2` |
 | `hellinger_euclidean_dist_matrix()` | `√partial_hellinger[i,j]` |
 
 ### BitPartials
 
-Required: `partial_jaccard() -> (Array2<u64>, Array2<u64>)` (inter, union), `partial_hamming() -> Array2<u64>`. Both additive across layers and partitions.
+Required: `partial_jaccard() -> (Array2<u64>, Array2<u64>)`, `partial_hamming() -> Array2<u64>`. Both additive across layers and partitions.
 
 ---
 
 ## Temp-file-backed types
 
-`MemoryBitVec` and `MemoryIntVec` are reserved for truly transient intra-method intermediates (e.g. a single `cmp_scalar` result that lives for one loop iteration). **All inter-function results use temp-file-backed types** so the OS can page them out under memory pressure. This matters in practice: processing dozens of layers × hundreds of partitions in parallel would otherwise accumulate gigabytes of live anonymous memory.
+**All inter-function results use temp-file-backed types** so the OS can page them out under memory pressure. This matters in practice: processing dozens of layers × hundreds of partitions in parallel would otherwise accumulate gigabytes of live anonymous memory.
 
 ### Lifecycle
 
 ```
 TempCompactIntVecBuilder::new(n)   →  writable mmap in TempDir
-     ↓  (set / add / count_bits / mask_with / …)
+     ↓  (inc_present_fast / inc_predicate_fast / add / mask_with / …)
  .freeze()                          →  TempCompactIntVec  (read-only mmap + TempDir)
      ↓  (optional)
  .make_persistent(path)             →  PersistentCompactIntVec  (permanent file)
@@ -587,7 +370,7 @@ TempCompactIntVecBuilder::new(n)   →  writable mmap in TempDir
 
 Same pattern for `TempBitVecBuilder` → `TempBitVec` → `PersistentBitVec`.
 
-**Drop order**: in `TempCompactIntVec { vec: PersistentCompactIntVec, _temp: TempDir }`, Rust drops fields in declaration order — `vec` (mmap) is released before `_temp` (directory) is deleted. No explicit `drop()` needed.
+**Drop order**: `TempCompactIntVec { vec: PersistentCompactIntVec, _temp: TempDir }` — Rust drops fields in declaration order. `vec` (mmap) released before `_temp` (directory deleted). No explicit `drop()` needed.
 
 ### TempCompactIntVec / TempCompactIntVecBuilder
 
@@ -603,9 +386,9 @@ pub(crate) struct TempCompactIntVecBuilder {
 }
 ```
 
-`TempCompactIntVec` implements `IntSlice` (full delegation to inner `PersistentCompactIntVec`).  
-`TempCompactIntVecBuilder` implements `IntSlice` + `IntSliceMut` (delegation to inner builder).  
-`make_persistent(path)` copies the temp file to `path` and opens it as `PersistentCompactIntVec`.
+`TempCompactIntVec`: read access via `get(slot)`, `sum()`, `iter()`, `view() -> IntSliceView<'_>`.
+
+`TempCompactIntVecBuilder`: full delegation to inner `PersistentCompactIntVecBuilder` — all bulk computation methods (`inc_present_fast`, `inc_predicate_fast`, `add`, `min`, `max`, `diff`, `mask_with`) are exposed as `pub(crate)`.
 
 ### TempBitVec / TempBitVecBuilder
 
@@ -621,9 +404,26 @@ pub(crate) struct TempBitVecBuilder {
 }
 ```
 
-`TempBitVec` implements `BitSlice`.  
-`TempBitVecBuilder` implements `BitSlice` + `BitSliceMut`.  
-`make_persistent(path)` copies the temp file and opens as `PersistentBitVec`.
+`TempBitVec`: read access via `get(slot)`, `count_ones()`, `view() -> BitSliceView<'_>`, `iter()`.
+
+`TempBitVecBuilder`: exposes `set(slot, bool)`, `or(BitSliceView)`, and:
+
+```rust
+pub(crate) fn or_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool)
+```
+
+`or_where` — two passes, no intermediate allocation:
+
+```
+Pass 1 — primary bytes, O(n):
+  for slot in 0..n:
+    b = col.primary_bytes()[slot]
+    if b < 255 AND pred(b as u32): self.set(slot, true)
+
+Pass 2 — overflow, O(k):
+  for (slot, val) in col.overflow_entries():
+    if pred(val): self.set(slot, true)
+```
 
 ---
 
@@ -635,74 +435,76 @@ pub(crate) struct TempBitVecBuilder {
 pub struct ColGroup { pub name: String, pub indices: Vec<usize> }
 ```
 
-Defined **once at the index level** from column metadata. Valid in all matrices of all layers and partitions because column structure is identical across the entire hierarchy (same samples/genomes everywhere; only rows = kmer slots are partitioned).
-
-`ColGroup` is passed by reference unchanged to any matrix — no index translation.
+Defined **once at the index level** from column metadata. Valid in all matrices of all layers and partitions — column structure is identical across the entire hierarchy; only rows (kmer slots) are partitioned.
 
 ### Composition axis
 
-- **Across partitions**: kmer space is partitioned → partial results are **concatenated** (disjoint kmer ranges).
-- **Across layers**: same kmer space, different counts → partial results are **aggregated** (add, OR, etc.).
+- **Across partitions**: kmer space is partitioned → partial results **concatenated** (disjoint kmer ranges).
+- **Across layers**: same kmer space, different counts → partial results **aggregated** (add, OR, etc.).
 
 ### MatrixGroupOps
 
-Group operations live on the matrix and expose only **additive intermediates** backed by temp files. Predicates (final thresholds → `MemoryBitVec`) are applied at the index level after accumulation.
+Five required primitives + two default methods derived from them. All return temp-file-backed types.
 
 ```rust
 pub trait MatrixGroupOps {
+    // required
     fn partial_group_presence_count(&self, g: &ColGroup, threshold: u32)
         -> io::Result<TempCompactIntVec>;
-
     fn partial_group_sum(&self, g: &ColGroup)
         -> io::Result<TempCompactIntVec>;
-
     fn partial_group_any(&self, g: &ColGroup, threshold: u32)
         -> io::Result<TempBitVec>;
+    fn partial_group_min(&self, g: &ColGroup)
+        -> io::Result<TempCompactIntVec>;
+    fn partial_group_max(&self, g: &ColGroup)
+        -> io::Result<TempCompactIntVec>;
+
+    // defaults derived from partial_group_presence_count
+    fn partial_group_all(&self, g: &ColGroup, threshold: u32)
+        -> io::Result<TempBitVec>;   // slot=1 iff count == g.indices.len()
+    fn partial_group_none(&self, g: &ColGroup, threshold: u32)
+        -> io::Result<TempBitVec>;   // slot=1 iff count == 0
 }
 ```
 
-Implemented for both `PersistentCompactIntMatrix` and `PersistentBitMatrix`. For bit matrices, `partial_group_sum` delegates to `partial_group_presence_count(g, 1)` since values are 0/1.
+Implemented for both `PersistentCompactIntMatrix` and `PersistentBitMatrix`.
+
+For **bit matrices**: values are 0/1, so `partial_group_sum` = `partial_group_presence_count(g, 1)`; `partial_group_min` is AND (set first column then mask-with remaining); `partial_group_max` is OR via `partial_group_any` + `inc_present`.
 
 **`partial_group_presence_count` — chunking for large groups:**
 
-When `g.indices.len() < 255`, per-slot counts fit in a raw `u8` — fast path: accumulate directly into `primary_bytes_mut()` using `inc_primary_bits`, then `freeze()`. No overflow map needed.
+When `g.indices.len() < 255`: per-slot counts stay within `u8` range. Use `inc_present_fast` (bit) or `inc_predicate_fast(col_view(c), |v| v >= threshold)` (int) — raw u8 increment, no overflow entry written.
 
-When `g.indices.len() ≥ 255`, process in chunks of 254 columns — each chunk stays within `u8` range — then add chunks into a running `TempCompactIntVecBuilder` accumulator via `IntSliceMut::add`. This keeps peak memory proportional to one partition, not the number of columns × partitions.
+When `g.indices.len() ≥ 255`: process in chunks of 254 columns, accumulate via `.add(chunk_frozen.view())`.
 
-```
-fast path (< 255 cols):
-  builder = TempCompactIntVecBuilder::new(n)
-  for c in group:
-    mask = col_view(c).cmp_scalar(|v| v >= threshold)  // MemoryBitVec
-    inc_primary_bits(primary_bytes_mut, mask)           // u8 safe
-  builder.freeze()
+**`partial_group_min` (int matrix)**: copy first column via `.add(col_view(first))` (start from 0 ⇒ copy), then `.min(col_view(c))` for remaining.
 
-slow path (≥ 255 cols):
-  result = TempCompactIntVecBuilder::new(n)
-  for chunk in group.chunks(254):
-    chunk_builder = TempCompactIntVecBuilder::new(n)
-    inc_primary_bits(chunk_builder, …)
-    chunk_frozen = chunk_builder.freeze()
-    IntSliceMut::add(&mut result, &chunk_frozen)
-  result.freeze()
+**`partial_group_max` (int matrix)**: `.max(col_view(c))` for all columns (start from 0 ⇒ first column acts as copy).
+
+**`partial_group_any`** uses `or_where` on `TempBitVecBuilder` (two-pass: primary bytes then overflow entries).
+
+**`partial_group_all` / `partial_group_none`** (default): call `partial_group_presence_count`, then iterate slots to produce the bit result. O(n) extra pass, not chunked.
+
+### add_col_from — matrix builder integration
+
+Both matrix builders accept temp-file results directly:
+
+```rust
+// PersistentBitMatrixBuilder
+fn add_col_from(&mut self, src: &TempBitVec)         -> io::Result<()>
+fn add_col_from_int(&mut self, src: &TempCompactIntVec) -> io::Result<()>  // nonzero → 1
+
+// PersistentCompactIntMatrixBuilder
+fn add_col_from(&mut self, src: &TempCompactIntVec)  -> io::Result<()>
+fn add_col_from_bit(&mut self, src: &TempBitVec)     -> io::Result<()>  // bit → 0/1 u32
 ```
 
-Non-additive predicates (`group_all`, `group_at_least(k)`) are **not** on the matrix — composed at the index level:
+`add_col_from` copies the temp file to the matrix directory and increments `n_cols`; `close()` writes `meta.json` with the final column count. No separate `write_meta` step needed.
 
-```
-// "present in >= 2 ingroup columns with count >= 3, absent from all outgroup"
-let presence = layers.map(|l| l.partial_group_presence_count(&ingroup, 3)?).add_all()?;
-let in_mask  = presence.geq(2);
+### mask_with
 
-let out_sum  = layers.map(|l| l.partial_group_sum(&outgroup)?).add_all()?;
-let out_mask = out_sum.leq(0);
-
-let mask = in_mask & &out_mask;    // BitSliceMut::and — O(n/64)
-```
-
-### mask_with (IntSliceMut)
-
-Provided method on `IntSliceMut`. Zeros every slot where the corresponding mask bit is 0. Iterates only zero bits — O(n_zeros), O(1) when mask is all-ones.
+Direct method on `PersistentCompactIntVecBuilder` (and delegation via `TempCompactIntVecBuilder`). Zeros every slot where the corresponding mask bit is 0. Iterates only zero bits — O(n_zeros), O(1) when mask is all-ones.
 
 ```
 for (w_idx, word) in mask.words():
@@ -711,7 +513,7 @@ for (w_idx, word) in mask.words():
   while zeros != 0:
     bit = trailing_zeros(zeros)
     s = w_idx * 64 + bit
-    if primary[s] != 0: self.set(s, 0)   // clears overflow entry too
+    if primary[s] != 0: set(s, 0)   // clears overflow entry too
     zeros &= zeros − 1
 ```
 

docmd/implementation/obitaxonomy.md

@@ -0,0 +1,143 @@
+# `obitaxonomy` — taxonomy concept paths
+
+`obitaxonomy` is a dependency-free crate that defines a typed representation
+of hierarchical concept paths (taxonomic or otherwise) stored in genome metadata.
+
+---
+
+## Concept path syntax
+
+A concept path is stored as a metadata value with the prefix `taxonomy:/`:
+
+```
+taxonomy:/enterobacteriaceae@family/Escherichia@genus/Escherichia coli@species
+```
+
+Structure:
+
+- The `taxonomy:/` prefix is the type discriminator. Any metadata value starting
+  with it is parsed as a `TaxPath`; all others remain plain strings.
+- The remainder is one or more `/`-separated segments.
+- Each segment is `name` or `name@rank`, where `rank` is a label for the
+  taxonomic level (e.g. `family`, `genus`, `species`).
+- Rank annotations are **optional per segment** and can be mixed freely.
+- Spaces are allowed in both names and ranks.
+
+### Reserved character
+
+`@` is reserved throughout the taxonomy system and may **not** appear in:
+
+| Context | Constraint |
+|---------|------------|
+| Segment name | forbidden |
+| Rank/class label | forbidden |
+| Metadata key names | forbidden (used as `key@rank` in predicate syntax) |
+
+`@` is freely allowed in plain-text metadata values (non-taxonomy).
+
+### Parse errors
+
+| Condition | Error |
+|-----------|-------|
+| Value does not start with `taxonomy:/` | `MissingPrefix` |
+| No segments after the prefix | `EmptyPath` |
+| Segment with empty name (consecutive `/`) | `EmptySegmentName` |
+| Segment with trailing `@` and no rank (`name@`) | `EmptyRankName` |
+| Segment with more than one `@` | `AmbiguousRank` |
+
+---
+
+## Public API
+
+### `TaxSegment`
+
+A single node: a name and an optional rank.
+
+```rust
+seg.name()            // &str
+seg.rank()            // Option<&str>
+seg.to_string()       // "name" or "name@rank"
+TaxSegment::parse(s)  // Result<TaxSegment, TaxError>
+```
+
+### `TaxPath`
+
+```rust
+TaxPath::parse(s)               // Result<TaxPath, TaxError>
+path.segments()                 // &[TaxSegment]
+path.depth()                    // usize — number of segments
+path.is_ancestor_of(&other)     // bool — prefix match by name, ranks ignored
+path.name_at_rank("genus")      // Option<&str>
+path.to_string()                // reconstructs "taxonomy:/…"
+```
+
+`is_ancestor_of` compares segment **names** only — rank annotations are
+informational and do not affect the ancestry relation.
+
+```rust
+let a: TaxPath = "taxonomy:/Enterobacteriaceae@family/Escherichia@genus".parse()?;
+let b: TaxPath = "taxonomy:/Enterobacteriaceae@family/Escherichia@genus/Escherichia coli@species".parse()?;
+
+assert!(a.is_ancestor_of(&b));   // true
+assert!(b.is_ancestor_of(&a));   // false
+assert!(a.is_ancestor_of(&a));   // true  (equal ⇒ ancestor)
+
+assert_eq!(b.name_at_rank("species"), Some("Escherichia coli"));
+assert_eq!(b.name_at_rank("genus"),   Some("Escherichia"));
+assert_eq!(b.name_at_rank("order"),   None);
+```
+
+---
+
+## Integration with `GenomeInfo`
+
+At index load time, every metadata value is inspected once:
+
+- Starts with `taxonomy:/` → parsed into `TaxPath`, stored in `genome.taxonomy`.
+- Otherwise → kept as-is in `genome.meta`.
+
+```rust
+struct GenomeInfo {
+    label:    String,
+    meta:     HashMap<String, String>,    // plain text metadata
+    taxonomy: HashMap<String, TaxPath>,   // parsed taxonomy metadata
+}
+```
+
+The raw string is not duplicated. `TaxPath::to_string()` reconstructs the
+original value losslessly for serialisation.
+
+---
+
+## Predicate operators (in `filter` / `select`)
+
+Path predicates use the `~` / `!~` operators. The **stored value** always starts
+with `/` (rooted path); the **query pattern** does not need to.
+
+### Path pattern syntax
+
+| Pattern | Semantics |
+|---------|-----------|
+| `A/B` | contiguous sub-path A then B, anywhere in the value |
+| `/A/B` | value starts with A then B (start-anchored) |
+| `A/B$` | value ends with A then B (end-anchored) |
+| `/A/B$` | value is exactly A then B (fully anchored) |
+| `A@x/B` | A with class `x` followed by B with any class |
+| `A@x/B@y` | A with class `x` followed by B with class `y` |
+
+A segment pattern without `@` matches the segment name regardless of its stored class.
+
+### Rank-aware queries
+
+```
+key@rank=value
+```
+
+| Predicate form | Semantics |
+|----------------|-----------|
+| `key@rank=value` | genome's `key` has `value` at rank `rank` |
+| `key@rank!=value` | does not |
+| `key@rank=v1\|v2` | value at `rank` is `v1` or `v2` |
+
+`~` combined with `@rank` on the key (e.g. `key@genus~pattern`) is not defined
+and is rejected at parse time.

mkdocs.yml

@@ -53,6 +53,7 @@ nav:
       - Merge parallelism & memory: implementation/merge_parallelism.md
       - Kmer filtering: implementation/filtering.md
       - Select command: implementation/select.md
+      - obitaxonomy crate: implementation/obitaxonomy.md
   - Architecture:
       - Sequences: architecture/sequences/invariant.md
       - Kmer index: architecture/index_architecture.md

src/Cargo.lock

@@ -1704,7 +1704,7 @@ dependencies = [
 
 [[package]]
 name = "obikmer"
-version = "0.1.0"
+version = "1.1.38"
 dependencies = [
  "clap",
  "csv",
@@ -1722,6 +1722,7 @@ dependencies = [
  "obiskbuilder",
  "obiskio",
  "obisys",
+ "obitaxonomy",
  "pprof",
  "rayon",
  "serde_json",
@@ -1853,6 +1854,10 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "obitaxonomy"
+version = "0.1.0"
+
 [[package]]
 name = "object"
 version = "0.37.3"

src/Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
 resolver = "3"
-members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obikpartitionner","obiskio","obidebruinj","obilayeredmap", "obicompactvec", "obisys", "obikindex"]
+members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obikpartitionner","obiskio","obidebruinj","obilayeredmap", "obicompactvec", "obisys", "obikindex", "obitaxonomy"]
 [profile.release]
 debug = 1

src/obicompactvec/src/bitmatrix.rs

@@ -1,5 +1,5 @@
 use std::fs::{self, File};
-use std::io::{self, BufWriter, Write as _};
+use std::io::{self, BufWriter, Read as _, Write as _};
 use std::path::{Path, PathBuf};
 
 use memmap2::Mmap;
@@ -7,13 +7,12 @@ use ndarray::{Array1, Array2};
 use rayon::prelude::*;
 
 use crate::bitvec::{PersistentBitVec, PersistentBitVecBuilder};
-use crate::colgroup::{ColGroup, MatrixGroupOps, inc_primary_bits};
-use crate::memoryvec::MemoryBitVec;
-use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
-use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
-use crate::traits::{BitSlice, BitSliceMut, IntSliceMut};
+use crate::colgroup::{ColGroup, MatrixGroupOps};
 use crate::layer_meta::LayerMeta;
 use crate::meta::MatrixMeta;
+use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
+use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
+use crate::views::BitSliceView;
 
 fn col_path(dir: &Path, col: usize) -> PathBuf {
     dir.join(format!("col_{col:06}.pbiv"))
@@ -143,18 +142,14 @@ impl PackedBitMatrix {
         unsafe { std::slice::from_raw_parts(ptr, nw) }
     }
 
-    pub(crate) fn col_slice(&self, c: usize) -> PackedCol<'_> {
-        PackedCol { words: self.col_words(c), n: self.n_rows }
+    pub(crate) fn col_slice(&self, c: usize) -> BitSliceView<'_> {
+        BitSliceView::new(self.col_words(c), self.n_rows)
     }
 
     pub(crate) fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentBitVecBuilder> {
         PersistentBitVecBuilder::from_raw_bytes(self.col_bytes(c), self.n_rows, path)
     }
 
-    pub(crate) fn col_as_memory(&self, c: usize) -> MemoryBitVec {
-        MemoryBitVec::from(&self.col_slice(c))
-    }
-
     pub(crate) fn count_ones(&self) -> Array1<u64> {
         Array1::from_vec(
             (0..self.n_cols).into_par_iter()
@@ -165,60 +160,54 @@ impl PackedBitMatrix {
 
     pub(crate) fn partial_jaccard_dist_matrix(&self) -> (Array2<u64>, Array2<u64>) {
         pairwise2_matrix(self.n_cols, |i, j| {
-            self.col_slice(i).partial_jaccard_dist(&self.col_slice(j))
+            self.col_slice(i).partial_jaccard_dist(self.col_slice(j))
         })
     }
 
     pub(crate) fn partial_hamming_dist_matrix(&self) -> Array2<u64> {
         pairwise_matrix(self.n_cols, |i, j| {
-            self.col_slice(i).hamming_dist(&self.col_slice(j))
+            self.col_slice(i).hamming_dist(self.col_slice(j))
         })
     }
 }
 
-pub(crate) struct PackedCol<'a> {
-    words: &'a [u64],
-    n: usize,
-}
-
-impl BitSlice for PackedCol<'_> {
-    fn len(&self) -> usize { self.n }
-    fn words(&self) -> &[u64] { self.words }
-}
-
-// ── BitColView — uniform column access across Columnar and Packed ─────────────
-
-enum BitColViewInner<'a> {
-    Columnar(&'a PersistentBitVec),
-    Packed(PackedCol<'a>),
-}
-
-/// Opaque column view returned by [`PersistentBitMatrix::col_view`].
-/// Implements [`BitSlice`] uniformly for both Columnar and Packed matrix formats.
-pub struct BitColView<'a>(BitColViewInner<'a>);
-
-impl BitSlice for BitColView<'_> {
-    fn len(&self) -> usize {
-        match &self.0 { BitColViewInner::Columnar(c) => c.len(), BitColViewInner::Packed(c) => c.len() }
-    }
-    fn words(&self) -> &[u64] {
-        match &self.0 { BitColViewInner::Columnar(c) => c.words(), BitColViewInner::Packed(c) => c.words() }
-    }
+/// Reads just the `n_cols` field from an existing packed matrix's header,
+/// without mapping the file. Used by `pack_bit_matrix` to tell a genuinely
+/// complete pack from a stale one that predates a later column-widening.
+fn packed_bit_matrix_n_cols(path: &Path) -> io::Result<usize> {
+    let mut f = File::open(path)?;
+    let mut header = [0u8; PBMX_HEADER];
+    f.read_exact(&mut header)?;
+    Ok(u64::from_le_bytes(header[16..24].try_into().unwrap()) as usize)
 }
 
 /// Build `presence/matrix.pbmx` from existing `col_*.pbiv` files.
 pub fn pack_bit_matrix(dir: &Path) -> io::Result<()> {
     let packed_path = dir.join("matrix.pbmx");
-    if packed_path.exists() {
-        // Matrix complete; remove any leftover column files from a killed cleanup.
-        if let Ok(meta) = MatrixMeta::load(dir) {
-            for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
-            let _ = fs::remove_file(dir.join("meta.json"));
+
+    let meta = match MatrixMeta::load(dir) {
+        Ok(meta) => meta,
+        Err(e) => {
+            // No columnar data pending: either this layer was already
+            // packed and cleaned up (matrix.pbmx complete, nothing left to
+            // do), or genuinely nothing was ever written here.
+            return if packed_path.exists() { Ok(()) } else { Err(e) };
         }
+    };
+
+    // A `matrix.pbmx` can already exist here even though columnar data is
+    // still pending — e.g. copied verbatim from a merge's base source
+    // before this layer was widened with more genome columns (see
+    // `obikpartitionner::merge_partition`). Only skip (re-)packing if the
+    // existing file already reflects the current column count; otherwise
+    // the columnar files are newer and must be (re-)packed, overwriting the
+    // stale one — never silently discarded as "leftover cleanup".
+    if packed_bit_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
+        for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
+        let _ = fs::remove_file(dir.join("meta.json"));
         return Ok(());
     }
 
-    let meta = MatrixMeta::load(dir)?;
     let n_cols = meta.n_cols;
 
     // Compute offsets from file sizes — no column data loaded into RAM.
@@ -321,34 +310,37 @@ impl PersistentBitMatrix {
         }
     }
 
-    pub fn col_view(&self, c: usize) -> BitColView<'_> {
+    pub fn col_view(&self, c: usize) -> BitSliceView<'_> {
         match self {
-            Self::Columnar(m) => BitColView(BitColViewInner::Columnar(m.col(c))),
-            Self::Packed(m)   => BitColView(BitColViewInner::Packed(m.col_slice(c))),
+            Self::Columnar(m) => m.col(c).view(),
+            Self::Packed(m)   => m.col_slice(c),
             Self::Implicit { .. } => panic!("col_view() not available on Implicit PersistentBitMatrix"),
         }
     }
 
+    /// Column-major point lookup: value at column `c`, slot `slot`, as 0/1.
+    ///
+    /// Unlike [`col_view`](Self::col_view), this never panics on `Implicit`
+    /// (every column reads as present, per the mono-genome fast path) — safe
+    /// to call for any `c < self.n_cols()`.
+    pub fn get(&self, c: usize, slot: usize) -> u32 {
+        match self {
+            Self::Columnar(m) => m.col(c).get(slot) as u32,
+            Self::Packed(m)   => m.col_slice(c).get(slot) as u32,
+            Self::Implicit { .. } => 1,
+        }
+    }
+
     pub fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentBitVecBuilder> {
         match self {
             Self::Columnar(m) => PersistentBitVecBuilder::build_from(m.col(c), path),
             Self::Packed(m)   => m.col_persist(c, path),
             Self::Implicit { n_rows, .. } => {
-                let mut b = PersistentBitVecBuilder::new(*n_rows, path)?;
-                b.not();
-                Ok(b)
+                PersistentBitVecBuilder::new_ones(*n_rows, path)
             }
         }
     }
 
-    pub fn col_as_memory(&self, c: usize) -> MemoryBitVec {
-        match self {
-            Self::Columnar(m) => MemoryBitVec::from(m.col(c)),
-            Self::Packed(m)   => m.col_as_memory(c),
-            Self::Implicit { n_rows, .. } => MemoryBitVec::ones(*n_rows),
-        }
-    }
-
     pub fn row(&self, slot: usize) -> Box<[bool]> {
         match self {
             Self::Columnar(m)             => m.row(slot),
@@ -445,6 +437,26 @@ impl PersistentBitMatrixBuilder {
         PersistentBitVecBuilder::new(self.n, &path)
     }
 
+    pub fn add_col_ones(&mut self) -> io::Result<PersistentBitVecBuilder> {
+        let path = col_path(&self.dir, self.n_cols);
+        self.n_cols += 1;
+        PersistentBitVecBuilder::new_ones(self.n, &path)
+    }
+
+    pub fn add_col_from(&mut self, src: &TempBitVec) -> io::Result<()> {
+        src.make_persistent(&col_path(&self.dir, self.n_cols))?;
+        self.n_cols += 1;
+        Ok(())
+    }
+
+    pub fn add_col_from_int(&mut self, src: &TempCompactIntVec) -> io::Result<()> {
+        let path = col_path(&self.dir, self.n_cols);
+        self.n_cols += 1;
+        let mut b = PersistentBitVecBuilder::new(self.n, &path)?;
+        b.or_where(src.view(), |v| v > 0);
+        b.close()
+    }
+
     pub fn close(self) -> io::Result<()> {
         MatrixMeta { n: self.n, n_cols: self.n_cols }.save(&self.dir)
     }
@@ -458,27 +470,19 @@ impl MatrixGroupOps for PersistentBitMatrix {
         let n = self.n();
         if g.indices.len() < 255 {
             let mut builder = TempCompactIntVecBuilder::new(n)?;
-            {
-                let primary = builder.primary_bytes_mut();
-                for &c in &g.indices {
-                    let mbv = MemoryBitVec::from(&self.col_view(c));
-                    inc_primary_bits(primary, &mbv);
-                }
+            for &c in &g.indices {
+                builder.inc_present_fast(self.col_view(c));
             }
             builder.freeze()
         } else {
             let mut result = TempCompactIntVecBuilder::new(n)?;
             for chunk in g.indices.chunks(254) {
-                let mut chunk_builder = TempCompactIntVecBuilder::new(n)?;
-                {
-                    let primary = chunk_builder.primary_bytes_mut();
-                    for &c in chunk {
-                        let mbv = MemoryBitVec::from(&self.col_view(c));
-                        inc_primary_bits(primary, &mbv);
-                    }
+                let mut chunk_b = TempCompactIntVecBuilder::new(n)?;
+                for &c in chunk {
+                    chunk_b.inc_present_fast(self.col_view(c));
                 }
-                let chunk_frozen = chunk_builder.freeze()?;
-                IntSliceMut::add(&mut result, &chunk_frozen);
+                let frozen = chunk_b.freeze()?;
+                result.add(frozen.view());
             }
             result.freeze()
         }
@@ -493,10 +497,30 @@ impl MatrixGroupOps for PersistentBitMatrix {
         let n = self.n();
         let mut result = TempBitVecBuilder::new(n)?;
         for &c in &g.indices {
-            result.or(&self.col_view(c));
+            result.or(self.col_view(c));
         }
         result.freeze()
     }
+
+    fn partial_group_min(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
+        // min of 0/1 values = AND: 1 only if ALL columns are 1
+        let n = self.n();
+        let mut result = TempCompactIntVecBuilder::new(n)?;
+        if let Some((&first, rest)) = g.indices.split_first() {
+            result.inc_present_fast(self.col_view(first));
+            for &c in rest { result.mask_with(self.col_view(c)); }
+        }
+        result.freeze()
+    }
+
+    fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
+        // max of 0/1 values = OR: 1 if any column is 1
+        let any = self.partial_group_any(g, 1)?;
+        let n = any.len();
+        let mut result = TempCompactIntVecBuilder::new(n)?;
+        result.inc_present(any.view());
+        result.freeze()
+    }
 }
 
 // ── Shared matrix helpers (also used by intmatrix.rs) ─────────────────────────
@@ -513,17 +537,26 @@ where T: Clone + Default {
 }
 
 /// Compute a symmetric `n×n` matrix in parallel by evaluating `f(i,j)` for
-/// all upper-triangle pairs. `T: Copy` avoids the `.clone()` needed for the
-/// lower-triangle mirror.
+/// all upper-triangle pairs, plus `f(i,i)` for the diagonal. `T: Copy` avoids
+/// the `.clone()` needed for the lower-triangle mirror.
+///
+/// The diagonal is *not* generally `T::default()`: for a self-comparison,
+/// `f(i,i)` is often the column's own weight (e.g. intersection-with-self —
+/// see `pairwise2_matrix`), not zero. Distance finalisations that need a
+/// zero diagonal (self-distance) already overwrite it explicitly.
 pub(crate) fn pairwise_matrix<T>(n: usize, f: impl Fn(usize, usize) -> T + Sync) -> Array2<T>
 where T: Copy + Default + Send {
     let results: Vec<(usize, usize, T)> = upper_pairs(n)
         .into_par_iter().map(|(i, j)| (i, j, f(i, j))).collect();
-    fill_symmetric(n, results.into_iter().map(|(i, j, v)| (i, j, v, v)))
+    let mut m = fill_symmetric(n, results.into_iter().map(|(i, j, v)| (i, j, v, v)));
+    for i in 0..n { m[[i, i]] = f(i, i); }
+    m
 }
 
 /// Same as `pairwise_matrix` but `f` returns two values that fill two
 /// symmetric matrices simultaneously (e.g. intersection + union for Jaccard).
+/// The diagonal is `f(i,i)` (e.g. a genome's kmer count intersected with
+/// itself), not `T::default()` — see `pairwise_matrix` for why that matters.
 pub(crate) fn pairwise2_matrix<T>(n: usize, f: impl Fn(usize, usize) -> (T, T) + Sync) -> (Array2<T>, Array2<T>)
 where T: Copy + Default + Send {
     let results: Vec<(usize, usize, T, T)> = upper_pairs(n)
@@ -536,5 +569,10 @@ where T: Copy + Default + Send {
         m0[[i, j]] = a; m0[[j, i]] = a;
         m1[[i, j]] = b; m1[[j, i]] = b;
     }
+    for i in 0..n {
+        let (a, b) = f(i, i);
+        m0[[i, i]] = a;
+        m1[[i, i]] = b;
+    }
     (m0, m1)
 }

src/obicompactvec/src/bitvec.rs

@@ -5,29 +5,25 @@ use std::path::{Path, PathBuf};
 use memmap2::{Mmap, MmapMut};
 
 use crate::reader::PersistentCompactIntVec;
+use crate::views::{BitSliceIter, BitSliceView, IntSliceView};
 
 const MAGIC: [u8; 4] = *b"PBIV";
 
 // Header: magic(4) + _pad(4) + n(8) = 16 bytes.
-// Data starts at offset 16, which is divisible by 8 → u64-aligned
-// (mmap base is page-aligned, 16 % 8 == 0).
+// Data starts at offset 16, u64-aligned (mmap base is page-aligned, 16 % 8 == 0).
 const HEADER_SIZE: usize = 16;
 
 #[inline]
-pub(crate) fn n_words(n: usize) -> usize {
-    n.div_ceil(64)
-}
+pub(crate) fn n_words(n: usize) -> usize { n.div_ceil(64) }
 
 #[inline]
-fn n_bytes_for_words(n: usize) -> usize {
-    n_words(n) * 8
-}
+fn n_bytes_for_words(n: usize) -> usize { n_words(n) * 8 }
 
-// ── Reader ────────────────────────────────────────────────────────────────────
+// ── PersistentBitVec ──────────────────────────────────────────────────────────
 
 pub struct PersistentBitVec {
     mmap: Mmap,
-    n: usize,
+    n:    usize,
     path: PathBuf,
 }
 
@@ -35,44 +31,49 @@ impl PersistentBitVec {
     pub fn open(path: &Path) -> io::Result<Self> {
         let mmap = unsafe { Mmap::map(&File::open(path)?)? };
         if mmap.len() < HEADER_SIZE {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidData,
-                "PBIV file too short",
-            ));
+            return Err(io::Error::new(io::ErrorKind::InvalidData, "PBIV file too short"));
         }
         if &mmap[0..4] != &MAGIC {
             return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PBIV magic"));
         }
         let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
-        Ok(Self {
-            mmap,
-            n,
-            path: path.to_path_buf(),
-        })
+        Ok(Self { mmap, n, path: path.to_path_buf() })
     }
 
-    pub fn path(&self) -> &Path {
-        &self.path
-    }
-    pub fn len(&self) -> usize {
-        self.n
-    }
-    pub fn is_empty(&self) -> bool {
-        self.n == 0
-    }
+    pub fn path(&self) -> &Path { &self.path }
+    pub fn len(&self)      -> usize { self.n }
+    pub fn is_empty(&self) -> bool  { self.n == 0 }
 
     pub fn get(&self, slot: usize) -> bool {
         (self.mmap[HEADER_SIZE + (slot >> 3)] >> (slot & 7)) & 1 != 0
     }
 
-    // SAFETY: mmap is page-aligned, HEADER_SIZE=16 is divisible by 8,
-    // so &mmap[HEADER_SIZE] is u64-aligned. Slice length is n_words * 8 bytes.
+    // SAFETY: mmap is page-aligned, HEADER_SIZE=16 divisible by 8 → u64-aligned.
     fn data_words(&self) -> &[u64] {
-        let nw = n_words(self.n);
+        let nw  = n_words(self.n);
         let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64;
         unsafe { std::slice::from_raw_parts(ptr, nw) }
     }
 
+    pub fn view(&self) -> BitSliceView<'_> {
+        BitSliceView::new(self.data_words(), self.n)
+    }
+
+    pub fn words(&self) -> &[u64] { self.data_words() }
+
+    pub fn count_ones(&self)  -> u64 { self.view().count_ones() }
+    pub fn count_zeros(&self) -> u64 { self.view().count_zeros() }
+
+    pub fn partial_jaccard_dist(&self, other: &PersistentBitVec) -> (u64, u64) {
+        self.view().partial_jaccard_dist(other.view())
+    }
+    pub fn jaccard_dist(&self, other: &PersistentBitVec) -> f64 {
+        self.view().jaccard_dist(other.view())
+    }
+    pub fn hamming_dist(&self, other: &PersistentBitVec) -> u64 {
+        self.view().hamming_dist(other.view())
+    }
+
     pub fn iter(&self) -> BitIter<'_> {
         BitIter { words: self.data_words(), slot: 0, n: self.n }
     }
@@ -81,40 +82,38 @@ impl PersistentBitVec {
 impl<'a> IntoIterator for &'a PersistentBitVec {
     type Item = bool;
     type IntoIter = BitIter<'a>;
-    fn into_iter(self) -> BitIter<'a> {
-        self.iter()
-    }
+    fn into_iter(self) -> BitIter<'a> { self.iter() }
 }
 
+// ── BitIter ───────────────────────────────────────────────────────────────────
+
 pub struct BitIter<'a> {
-    pub(crate) words: &'a [u64],
-    pub(crate) slot: usize,
-    pub(crate) n: usize,
+    words: &'a [u64],
+    slot:  usize,
+    n:     usize,
 }
 
 impl ExactSizeIterator for BitIter<'_> {}
 
 impl Iterator for BitIter<'_> {
     type Item = bool;
-
     fn next(&mut self) -> Option<bool> {
         if self.slot >= self.n { return None; }
         let v = (self.words[self.slot >> 6] >> (self.slot & 63)) & 1 != 0;
         self.slot += 1;
         Some(v)
     }
-
     fn size_hint(&self) -> (usize, Option<usize>) {
         let rem = self.n - self.slot;
         (rem, Some(rem))
     }
 }
 
-// ── Builder ───────────────────────────────────────────────────────────────────
+// ── PersistentBitVecBuilder ───────────────────────────────────────────────────
 
 pub struct PersistentBitVecBuilder {
     mmap: MmapMut,
-    n: usize,
+    n:    usize,
     path: PathBuf,
 }
 
@@ -122,13 +121,10 @@ impl PersistentBitVecBuilder {
     pub fn new(n: usize, path: &Path) -> io::Result<Self> {
         let file_size = HEADER_SIZE + n_bytes_for_words(n);
         let mut file = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(true)
-            .truncate(true)
+            .read(true).write(true).create(true).truncate(true)
             .open(path)?;
         file.write_all(&MAGIC)?;
-        file.write_all(&[0u8; 4])?; // padding
+        file.write_all(&[0u8; 4])?;
         file.write_all(&(n as u64).to_le_bytes())?;
         file.seek(SeekFrom::Start(0))?;
         file.set_len(file_size as u64)?;
@@ -136,9 +132,7 @@ impl PersistentBitVecBuilder {
         Ok(Self { mmap, n, path: path.to_path_buf() })
     }
 
-    /// Create a PBIV file from raw packed bit-bytes, zero-padding to the next word boundary.
-    /// `bytes` is `n.div_ceil(8)` bytes; `n` is the number of bits.
-    pub(crate) fn from_raw_bytes(bytes: &[u8], n: usize, path: &Path) -> io::Result<Self> {
+    pub fn from_raw_bytes(bytes: &[u8], n: usize, path: &Path) -> io::Result<Self> {
         let file_size = HEADER_SIZE + n_bytes_for_words(n);
         let file = OpenOptions::new()
             .read(true).write(true).create(true).truncate(true)
@@ -151,6 +145,33 @@ impl PersistentBitVecBuilder {
         Ok(Self { mmap, n, path: path.to_path_buf() })
     }
 
+    /// Create an all-ones bit vector of length `n` at `path`.
+    ///
+    /// More efficient than `new(n, path)` + `not()`: the data is written as
+    /// 0xFF bytes in a single sequential pass, with no intermediate all-zeros state.
+    pub fn new_ones(n: usize, path: &Path) -> io::Result<Self> {
+        let nw        = n_words(n);
+        let file_size = HEADER_SIZE + nw * 8;
+        let mut file  = OpenOptions::new()
+            .read(true).write(true).create(true).truncate(true)
+            .open(path)?;
+        file.write_all(&MAGIC)?;
+        file.write_all(&[0u8; 4])?;
+        file.write_all(&(n as u64).to_le_bytes())?;
+        file.write_all(&vec![0xFFu8; nw * 8])?;
+        file.seek(SeekFrom::Start(0))?;
+        file.set_len(file_size as u64)?;
+        let mut mmap = unsafe { MmapMut::map_mut(&file)? };
+        // Clear padding bits in the last word so trailing bits are always 0.
+        let rem = n % 64;
+        if rem != 0 {
+            let ptr   = mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64;
+            let words = unsafe { std::slice::from_raw_parts_mut(ptr, nw) };
+            words[nw - 1] &= (1u64 << rem) - 1;
+        }
+        Ok(Self { mmap, n, path: path.to_path_buf() })
+    }
+
     pub fn build_from(source: &PersistentBitVec, path: &Path) -> io::Result<Self> {
         fs::copy(source.path(), path)?;
         let file = OpenOptions::new().read(true).write(true).open(path)?;
@@ -159,44 +180,11 @@ impl PersistentBitVecBuilder {
         Ok(Self { mmap, n, path: path.to_path_buf() })
     }
 
-    pub fn len(&self) -> usize {
-        self.n
-    }
-    pub fn is_empty(&self) -> bool {
-        self.n == 0
-    }
-
-    pub fn get(&self, slot: usize) -> bool {
-        (self.mmap[HEADER_SIZE + (slot >> 3)] >> (slot & 7)) & 1 != 0
-    }
-
-    fn data_words(&self) -> &[u64] {
-        let nw = n_words(self.n);
-        let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64;
-        unsafe { std::slice::from_raw_parts(ptr, nw) }
-    }
-
-    // SAFETY: same alignment argument as PersistentBitVec::data_words.
-    fn data_words_mut(&mut self) -> &mut [u64] {
-        let nw = n_words(self.n);
-        let ptr = self.mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64;
-        unsafe { std::slice::from_raw_parts_mut(ptr, nw) }
-    }
-
-    /// Convert a count vector to a bit vector: bit set iff count >= threshold.
-    /// Fills u64 words directly from the count iterator — O(n), no bit-level set() overhead.
-    pub fn build_from_counts(
-        source: &PersistentCompactIntVec,
-        threshold: u32,
-        path: &Path,
-    ) -> io::Result<Self> {
+    pub fn build_from_counts(source: &PersistentCompactIntVec, threshold: u32, path: &Path) -> io::Result<Self> {
         let n = source.len();
         let file_size = HEADER_SIZE + n_bytes_for_words(n);
         let mut file = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(true)
-            .truncate(true)
+            .read(true).write(true).create(true).truncate(true)
             .open(path)?;
         file.write_all(&MAGIC)?;
         file.write_all(&[0u8; 4])?;
@@ -204,52 +192,157 @@ impl PersistentBitVecBuilder {
         file.seek(SeekFrom::Start(0))?;
         file.set_len(file_size as u64)?;
         let mut mmap = unsafe { MmapMut::map_mut(&file)? };
-
         {
-            let nw = n_words(n);
+            let nw  = n_words(n);
             let ptr = mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64;
             let words = unsafe { std::slice::from_raw_parts_mut(ptr, nw) };
             for (slot, count) in source.iter().enumerate() {
-                if count >= threshold {
-                    words[slot >> 6] |= 1u64 << (slot & 63);
-                }
+                if count >= threshold { words[slot >> 6] |= 1u64 << (slot & 63); }
             }
         }
-
         Ok(Self { mmap, n, path: path.to_path_buf() })
     }
 
-    /// Convert a count vector to a presence/absence bit vector (threshold = 1).
     pub fn build_from_presence(source: &PersistentCompactIntVec, path: &Path) -> io::Result<Self> {
         Self::build_from_counts(source, 1, path)
     }
 
-    pub fn close(self) -> io::Result<()> {
-        self.mmap.flush()
+    pub fn len(&self)      -> usize { self.n }
+    pub fn is_empty(&self) -> bool  { self.n == 0 }
+
+    pub fn get(&self, slot: usize) -> bool {
+        (self.mmap[HEADER_SIZE + (slot >> 3)] >> (slot & 7)) & 1 != 0
     }
 
-    /// Flush, close, and reopen as a read-only `PersistentBitVec`.
+    pub fn set(&mut self, slot: usize, value: bool) {
+        let bit = 1u64 << (slot & 63);
+        if value { self.data_words_mut()[slot >> 6] |=  bit; }
+        else     { self.data_words_mut()[slot >> 6] &= !bit; }
+    }
+
+    fn data_words(&self) -> &[u64] {
+        let nw  = n_words(self.n);
+        let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64;
+        unsafe { std::slice::from_raw_parts(ptr, nw) }
+    }
+
+    // SAFETY: same alignment argument as PersistentBitVec::data_words.
+    fn data_words_mut(&mut self) -> &mut [u64] {
+        let nw  = n_words(self.n);
+        let ptr = self.mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64;
+        unsafe { std::slice::from_raw_parts_mut(ptr, nw) }
+    }
+
+    pub fn view(&self) -> BitSliceView<'_> {
+        BitSliceView::new(self.data_words(), self.n)
+    }
+
+    pub fn words(&self) -> &[u64] { self.data_words() }
+
+    pub fn copy_from(&mut self, src: BitSliceView<'_>) {
+        assert_eq!(self.n, src.len(), "BitSliceView length mismatch");
+        self.data_words_mut().copy_from_slice(src.words());
+    }
+
+    pub fn and(&mut self, other: BitSliceView<'_>) {
+        assert_eq!(self.n, other.len(), "BitSliceView length mismatch");
+        for (w, &o) in self.data_words_mut().iter_mut().zip(other.words()) { *w &= o; }
+    }
+
+    pub fn or(&mut self, other: BitSliceView<'_>) {
+        assert_eq!(self.n, other.len(), "BitSliceView length mismatch");
+        for (w, &o) in self.data_words_mut().iter_mut().zip(other.words()) { *w |= o; }
+    }
+
+    pub fn xor(&mut self, other: BitSliceView<'_>) {
+        assert_eq!(self.n, other.len(), "BitSliceView length mismatch");
+        for (w, &o) in self.data_words_mut().iter_mut().zip(other.words()) { *w ^= o; }
+    }
+
+    pub fn not(&mut self) {
+        let rem   = self.n % 64;
+        let words = self.data_words_mut();
+        for w in words.iter_mut() { *w ^= u64::MAX; }
+        if rem != 0 {
+            if let Some(last) = words.last_mut() { *last &= (1u64 << rem) - 1; }
+        }
+    }
+
+    /// OR in bits at slots where `pred(col[slot])` is true.
+    pub fn or_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
+        assert_eq!(self.n, col.len(), "IntSliceView length mismatch");
+        let n = self.n;
+        let primary = col.primary_bytes();
+        let words = self.data_words_mut();
+        let nw = n_words(n);
+        for wi in 0..nw {
+            let base  = wi * 64;
+            let limit = (base + 64).min(n);
+            let mut mask = 0u64;
+            for bit in 0..(limit - base) {
+                let b = primary[base + bit];
+                if b < 255 && pred(b as u32) { mask |= 1u64 << bit; }
+            }
+            words[wi] |= mask;
+        }
+        for (slot, val) in col.overflow_entries() {
+            if pred(val) { words[slot >> 6] |= 1u64 << (slot & 63); }
+        }
+    }
+
+    /// Clear bits at slots where `pred(col[slot])` is false.
+    pub fn and_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
+        assert_eq!(self.n, col.len(), "IntSliceView length mismatch");
+        let n = self.n;
+        let primary = col.primary_bytes();
+        let words = self.data_words_mut();
+        let nw = n_words(n);
+        for wi in 0..nw {
+            let base  = wi * 64;
+            let limit = (base + 64).min(n);
+            let mut mask = 0u64;
+            for bit in 0..(limit - base) {
+                let b = primary[base + bit];
+                if b < 255 && !pred(b as u32) { mask |= 1u64 << bit; }
+            }
+            words[wi] &= !mask;
+        }
+        for (slot, val) in col.overflow_entries() {
+            if !pred(val) { words[slot >> 6] &= !(1u64 << (slot & 63)); }
+        }
+    }
+
+    /// Toggle bits at slots where `pred(col[slot])` is true.
+    pub fn xor_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
+        assert_eq!(self.n, col.len(), "IntSliceView length mismatch");
+        let n = self.n;
+        let primary = col.primary_bytes();
+        let words = self.data_words_mut();
+        let nw = n_words(n);
+        for wi in 0..nw {
+            let base  = wi * 64;
+            let limit = (base + 64).min(n);
+            let mut mask = 0u64;
+            for bit in 0..(limit - base) {
+                let b = primary[base + bit];
+                if b < 255 && pred(b as u32) { mask |= 1u64 << bit; }
+            }
+            words[wi] ^= mask;
+        }
+        for (slot, val) in col.overflow_entries() {
+            if pred(val) { words[slot >> 6] ^= 1u64 << (slot & 63); }
+        }
+    }
+
+    pub fn iter(&self) -> BitSliceIter<'_> {
+        self.view().iter()
+    }
+
+    pub fn close(self) -> io::Result<()> { self.mmap.flush() }
+
     pub fn finish(self) -> io::Result<PersistentBitVec> {
         let path = self.path.clone();
         self.close()?;
         PersistentBitVec::open(&path)
     }
 }
-
-// ── BitSlice / BitSliceMut impls ──────────────────────────────────────────────
-
-use crate::traits::{BitSlice, BitSliceMut};
-
-impl BitSlice for PersistentBitVec {
-    fn len(&self) -> usize { self.n }
-    fn words(&self) -> &[u64] { self.data_words() }
-}
-
-impl BitSlice for PersistentBitVecBuilder {
-    fn len(&self) -> usize { self.n }
-    fn words(&self) -> &[u64] { self.data_words() }
-}
-
-impl BitSliceMut for PersistentBitVecBuilder {
-    fn words_mut(&mut self) -> &mut [u64] { self.data_words_mut() }
-}

src/obicompactvec/src/builder.rs

@@ -7,53 +7,26 @@ use memmap2::MmapMut;
 
 use crate::format::{byte_count_nonzero, byte_sum, HEADER_SIZE, finalize_pciv, parse_overflow_entry};
 use crate::reader::PersistentCompactIntVec;
+use crate::views::{BitSliceView, IntSliceView};
 
 pub struct PersistentCompactIntVecBuilder {
-    path: PathBuf,
-    mmap: MmapMut,
-    n: usize,
+    path:     PathBuf,
+    mmap:     MmapMut,
+    n:        usize,
     overflow: HashMap<usize, u32>,
 }
 
 impl PersistentCompactIntVecBuilder {
-    /// Create a new, zero-filled PCIV at `path`. Primary is mmapped immediately.
     pub fn new(n: usize, path: &Path) -> io::Result<Self> {
-        let file = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(true)
-            .truncate(true)
-            .open(path)?;
-        file.set_len((HEADER_SIZE + n) as u64)?;
-        let mmap = unsafe { MmapMut::map_mut(&file)? };
-        Ok(Self {
-            path: path.to_path_buf(),
-            mmap,
-            n,
-            overflow: HashMap::new(),
-        })
-    }
-
-    /// Create from a [`MemoryIntVec`], copying primary bytes directly into the mmap.
-    /// O(n) memcpy + O(n_overflow) HashMap clone — no per-slot `set` overhead.
-    pub fn from_memory(src: &crate::memoryintvec::MemoryIntVec, path: &Path) -> io::Result<Self> {
-        let n = src.len();
         let file = OpenOptions::new()
             .read(true).write(true).create(true).truncate(true)
             .open(path)?;
         file.set_len((HEADER_SIZE + n) as u64)?;
-        let mut mmap = unsafe { MmapMut::map_mut(&file)? };
-        mmap[HEADER_SIZE..HEADER_SIZE + n].copy_from_slice(src.primary_bytes());
-        Ok(Self {
-            path: path.to_path_buf(),
-            mmap,
-            n,
-            overflow: src.overflow_map().clone(),
-        })
+        let mmap = unsafe { MmapMut::map_mut(&file)? };
+        Ok(Self { path: path.to_path_buf(), mmap, n, overflow: HashMap::new() })
     }
 
-    /// Create from raw primary bytes + an already-built overflow map (no per-slot overhead).
-    pub(crate) fn from_raw_primary(primary: &[u8], overflow: HashMap<usize, u32>, path: &Path) -> io::Result<Self> {
+    pub fn from_raw_primary(primary: &[u8], overflow: HashMap<usize, u32>, path: &Path) -> io::Result<Self> {
         let n = primary.len();
         let file = OpenOptions::new()
             .read(true).write(true).create(true).truncate(true)
@@ -64,40 +37,25 @@ impl PersistentCompactIntVecBuilder {
         Ok(Self { path: path.to_path_buf(), mmap, n, overflow })
     }
 
-    /// Copy `source`'s file to `path`, mmap the primary section, load overflow into RAM.
-    /// Avoids iterating all n slots: the file copy is OS-level, overflow loading is O(n_overflow).
     pub fn build_from(source: &PersistentCompactIntVec, path: &Path) -> io::Result<Self> {
         fs::copy(source.path(), path)?;
-
         let file = OpenOptions::new().read(true).write(true).open(path)?;
         let mmap = unsafe { MmapMut::map_mut(&file)? };
-
-        let n = source.len();
+        let n          = source.len();
         let n_overflow = u64::from_le_bytes(mmap[16..24].try_into().unwrap()) as usize;
         let data_offset = HEADER_SIZE + n;
-
         let mut overflow = HashMap::with_capacity(n_overflow);
         for i in 0..n_overflow {
             let (slot, value) = parse_overflow_entry(&mmap, data_offset, i);
             overflow.insert(slot, value);
         }
-
-        Ok(Self {
-            path: path.to_path_buf(),
-            mmap,
-            n,
-            overflow,
-        })
+        Ok(Self { path: path.to_path_buf(), mmap, n, overflow })
     }
 
-    /// Get the value at the given slot, handling overflow if necessary.
     pub fn get(&self, slot: usize) -> u32 {
         match self.mmap[HEADER_SIZE + slot] {
-            255 => *self
-                .overflow
-                .get(&slot)
-                .expect("sentinel without overflow entry"),
-            v => v as u32,
+            255 => *self.overflow.get(&slot).expect("sentinel without overflow entry"),
+            v   => v as u32,
         }
     }
 
@@ -111,15 +69,189 @@ impl PersistentCompactIntVecBuilder {
         }
     }
 
-    pub fn len(&self) -> usize {
-        self.n
+    pub fn len(&self)      -> usize { self.n }
+    pub fn is_empty(&self) -> bool  { self.n == 0 }
+
+    pub fn primary_bytes(&self)     -> &[u8]      { &self.mmap[HEADER_SIZE..HEADER_SIZE + self.n] }
+    pub fn primary_bytes_mut(&mut self) -> &mut [u8] { &mut self.mmap[HEADER_SIZE..HEADER_SIZE + self.n] }
+    pub fn clear_overflow(&mut self) { self.overflow.clear(); }
+
+    pub fn sum(&self) -> u64 {
+        byte_sum(&self.mmap[HEADER_SIZE..HEADER_SIZE + self.n], self.overflow.values().copied())
+    }
+    pub fn count_nonzero(&self) -> u64 {
+        byte_count_nonzero(&self.mmap[HEADER_SIZE..HEADER_SIZE + self.n])
     }
 
-    pub fn is_empty(&self) -> bool {
-        self.n == 0
+    pub fn view(&self) -> IntSliceView<'_> {
+        // Builder overflow is a HashMap, not sorted raw bytes — convert on the fly
+        // by collecting into a sorted vec and storing in a thread-local buffer.
+        // For read-back during building, just call get(slot) directly.
+        // view() is primarily useful AFTER freeze (on PersistentCompactIntVec).
+        // Here we expose it via a zero-alloc path: primary only, no overflow raw.
+        // Callers that need overflow_entries during building use overflow_entries().
+        let primary = &self.mmap[HEADER_SIZE..HEADER_SIZE + self.n];
+        IntSliceView::new(primary, &[], 0, self.n)
+    }
+
+    pub fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_ {
+        self.overflow.iter().map(|(&k, &v)| (k, v))
+    }
+
+    pub fn inc(&mut self, slot: usize) {
+        let v = self.get(slot);
+        self.set(slot, v.saturating_add(1));
+    }
+
+    // ── Computation methods ───────────────────────────────────────────────────
+
+    /// Increment one counter per 1-bit of `col`.  Safe for any group size.
+    pub fn inc_present(&mut self, col: BitSliceView<'_>) {
+        let n = self.n;
+        for (wi, &word) in col.words().iter().enumerate() {
+            if word == 0 { continue; }
+            let mut w = word;
+            while w != 0 {
+                let bit  = w.trailing_zeros() as usize;
+                let slot = wi * 64 + bit;
+                if slot < n { self.inc(slot); }
+                w &= w - 1;
+            }
+        }
+    }
+
+    /// Increment one counter per 1-bit of `col`, using raw u8 arithmetic.
+    /// Caller guarantees no counter will reach 255 (group size < 255).
+    pub fn inc_present_fast(&mut self, col: BitSliceView<'_>) {
+        {
+            let primary = self.primary_bytes_mut();
+            let n       = primary.len();
+            for (wi, &word) in col.words().iter().enumerate() {
+                if word == 0 { continue; }
+                let mut w = word;
+                while w != 0 {
+                    let bit  = w.trailing_zeros() as usize;
+                    let s    = wi * 64 + bit;
+                    if s < n { primary[s] += 1; }
+                    w &= w - 1;
+                }
+            }
+        }
+        debug_assert!(
+            !self.primary_bytes().contains(&255),
+            "sentinel 255 reached in inc_present_fast — group size must be < 255"
+        );
+    }
+
+    /// Two-pass: primary bytes then overflow.  Increments `self[slot]` for each
+    /// slot where `pred(col[slot])` is true.  Safe for any group size.
+    pub fn inc_predicate(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
+        let n = col.len();
+        for slot in 0..n {
+            let b = col.primary_bytes()[slot];
+            if b < 255 && pred(b as u32) {
+                self.inc(slot);
+            }
+        }
+        for (slot, val) in col.overflow_entries() {
+            if pred(val) { self.inc(slot); }
+        }
+    }
+
+    /// Fast two-pass: raw u8 arithmetic.  Caller guarantees no counter reaches 255.
+    pub fn inc_predicate_fast(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
+        let n = col.len();
+        {
+            let primary = self.primary_bytes_mut();
+            for slot in 0..n {
+                let b = col.primary_bytes()[slot];
+                if b < 255 && pred(b as u32) {
+                    primary[slot] += 1;
+                }
+            }
+        }
+        for (slot, val) in col.overflow_entries() {
+            if pred(val) { self.primary_bytes_mut()[slot] += 1; }
+        }
+        debug_assert!(
+            !self.primary_bytes().contains(&255),
+            "sentinel 255 reached in inc_predicate_fast — group size must be < 255"
+        );
+    }
+
+    pub fn add(&mut self, other: IntSliceView<'_>) {
+        let n = self.n;
+        for s in 0..n {
+            let sb = self.primary_bytes()[s];
+            let ob = other.primary_bytes()[s];
+            if sb < 255 && ob < 255 {
+                let sum = sb as u32 + ob as u32;
+                if sum < 255 { self.primary_bytes_mut()[s] = sum as u8; }
+                else         { self.set(s, sum); }
+            } else {
+                let sv = self.get(s);
+                let ov = other.get(s);
+                self.set(s, sv + ov);
+            }
+        }
+    }
+
+    pub fn min(&mut self, other: IntSliceView<'_>) {
+        let self_ov: Vec<(usize, u32)> = self.overflow_entries().collect();
+        let other_ov: HashMap<usize, u32> = other.overflow_entries().collect();
+        self.clear_overflow();
+        for (a, &b) in self.primary_bytes_mut().iter_mut().zip(other.primary_bytes()) {
+            if b < *a { *a = b; }
+        }
+        for (slot, self_val) in self_ov {
+            if let Some(&other_val) = other_ov.get(&slot) {
+                self.set(slot, self_val.min(other_val));
+            }
+        }
+    }
+
+    pub fn max(&mut self, other: IntSliceView<'_>) {
+        for (slot, other_val) in other.overflow_entries() {
+            let sv = self.get(slot);
+            self.set(slot, sv.max(other_val));
+        }
+        for (a, &b) in self.primary_bytes_mut().iter_mut().zip(other.primary_bytes()) {
+            if b > *a { *a = b; }
+        }
+    }
+
+    pub fn diff(&mut self, other: IntSliceView<'_>) {
+        let n = self.n;
+        for s in 0..n {
+            let sb = self.primary_bytes()[s];
+            let ob = other.primary_bytes()[s];
+            if sb < 255 {
+                self.primary_bytes_mut()[s] = if ob < 255 { sb.saturating_sub(ob) } else { 0 };
+            } else {
+                let sv = self.get(s);
+                let ov = if ob < 255 { ob as u32 } else { other.get(s) };
+                self.set(s, sv.saturating_sub(ov));
+            }
+        }
+    }
+
+    pub fn mask_with(&mut self, mask: BitSliceView<'_>) {
+        let n = self.n;
+        for (wi, &word) in mask.words().iter().enumerate() {
+            if word == u64::MAX { continue; }
+            let mut zeros = !word;
+            while zeros != 0 {
+                let bit = zeros.trailing_zeros() as usize;
+                let s   = wi * 64 + bit;
+                if s < n {
+                    let b = self.primary_bytes()[s];
+                    if b != 0 { self.set(s, 0); }
+                }
+                zeros &= zeros - 1;
+            }
+        }
     }
 
-    /// Flush the primary mmap, then write sorted overflow data + index and fix the header.
     pub fn close(self) -> io::Result<()> {
         self.mmap.flush()?;
         let Self { path, mmap, n, overflow } = self;
@@ -129,35 +261,9 @@ impl PersistentCompactIntVecBuilder {
         finalize_pciv(&path, n, &entries)
     }
 
-    /// Close and reopen as a read-only [`PersistentCompactIntVec`].
     pub fn finish(self) -> io::Result<PersistentCompactIntVec> {
         let path = self.path.clone();
         self.close()?;
         PersistentCompactIntVec::open(&path)
     }
 }
-
-// ── IntSlice / IntSliceMut impls ──────────────────────────────────────────────
-
-use crate::traits::{IntSlice, IntSliceMut};
-
-impl IntSlice for PersistentCompactIntVecBuilder {
-    fn len(&self) -> usize { self.n }
-    fn get(&self, slot: usize) -> u32 { self.get(slot) }
-    fn primary_bytes(&self) -> &[u8] { &self.mmap[HEADER_SIZE..HEADER_SIZE + self.n] }
-    fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_ {
-        self.overflow.iter().map(|(&k, &v)| (k, v))
-    }
-    fn sum(&self) -> u64 {
-        byte_sum(&self.mmap[HEADER_SIZE..HEADER_SIZE + self.n], self.overflow.values().copied())
-    }
-    fn count_nonzero(&self) -> u64 {
-        byte_count_nonzero(&self.mmap[HEADER_SIZE..HEADER_SIZE + self.n])
-    }
-}
-
-impl IntSliceMut for PersistentCompactIntVecBuilder {
-    fn set(&mut self, slot: usize, value: u32) { self.set(slot, value); }
-    fn primary_bytes_mut(&mut self) -> &mut [u8] { &mut self.mmap[HEADER_SIZE..HEADER_SIZE + self.n] }
-    fn clear_overflow(&mut self) { self.overflow.clear(); }
-}

src/obicompactvec/src/colgroup.rs

@@ -1,9 +1,7 @@
 use std::io;
 
-use crate::memoryvec::MemoryBitVec;
-use crate::tempbitvec::TempBitVec;
+use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
 use crate::tempintvec::TempCompactIntVec;
-use crate::traits::BitSlice;
 
 // ── ColGroup ──────────────────────────────────────────────────────────────────
 
@@ -25,12 +23,14 @@ impl ColGroup {
 
 // ── MatrixGroupOps ────────────────────────────────────────────────────────────
 
-/// Per-matrix group aggregations that return **additive intermediates**.
+/// Per-matrix group aggregations.
 ///
-/// Results must be composed by the caller (concat across partitions, add across
-/// layers) before applying final predicates (`geq`, `leq`, …).  Non-additive
-/// predicates like `group_all` or `group_at_least(k)` are intentionally absent
-/// — they are derived at the index level from these intermediates.
+/// `partial_group_presence_count`, `partial_group_sum`, `partial_group_any`,
+/// `partial_group_min`, `partial_group_max` are the primitives; each impl must
+/// provide all five.
+///
+/// `partial_group_all` and `partial_group_none` have default implementations
+/// derived from `partial_group_presence_count` and should rarely need overriding.
 pub trait MatrixGroupOps {
     /// Per-slot count of group columns whose value ≥ `threshold`.
     fn partial_group_presence_count(&self, g: &ColGroup, threshold: u32) -> io::Result<TempCompactIntVec>;
@@ -38,25 +38,100 @@ pub trait MatrixGroupOps {
     /// Per-slot sum of values across all group columns.
     fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec>;
 
-    /// Per-slot OR: true if any group column has value ≥ `threshold`.
+    /// Per-slot OR: 1 if any group column has value ≥ `threshold`.
     fn partial_group_any(&self, g: &ColGroup, threshold: u32) -> io::Result<TempBitVec>;
+
+    /// Per-slot min value across all group columns (0 if group is empty).
+    fn partial_group_min(&self, g: &ColGroup) -> io::Result<TempCompactIntVec>;
+
+    /// Per-slot max value across all group columns (0 if group is empty).
+    fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec>;
+
+    /// Per-slot AND: 1 if ALL group columns have value ≥ `threshold`.
+    fn partial_group_all(&self, g: &ColGroup, threshold: u32) -> io::Result<TempBitVec> {
+        let counts = self.partial_group_presence_count(g, threshold)?;
+        let n = counts.len();
+        let n_required = g.indices.len() as u32;
+        let mut b = TempBitVecBuilder::new(n)?;
+        b.or_where(counts.view(), |v| v >= n_required);
+        b.freeze()
+    }
+
+    /// Per-slot NOR: 1 if NO group column has value ≥ `threshold`.
+    fn partial_group_none(&self, g: &ColGroup, threshold: u32) -> io::Result<TempBitVec> {
+        let counts = self.partial_group_presence_count(g, threshold)?;
+        let n = counts.len();
+        let mut b = TempBitVecBuilder::new(n)?;
+        b.or_where(counts.view(), |v| v == 0);
+        b.freeze()
+    }
 }
 
-// ── Internal helper ───────────────────────────────────────────────────────────
+// ── FilterMask — expression tree for column-based slot filters ────────────────
 
-/// Iterate 1-bits of a `MemoryBitVec` and increment the corresponding raw
-/// byte.  Caller must guarantee that no counter will reach 255 (group size
-/// < 255 columns), so that incrementing `u8` is safe and no sentinel is
-/// accidentally written.
-pub(crate) fn inc_primary_bits(primary: &mut [u8], mask: &MemoryBitVec) {
-    let n = primary.len();
-    for (wi, &word) in mask.words().iter().enumerate() {
-        let mut w = word;
-        while w != 0 {
-            let bit = w.trailing_zeros() as usize;
-            let s = wi * 64 + bit;
-            if s < n { primary[s] += 1; }
-            w &= w - 1;
+/// A composable filter expression that can be evaluated against a matrix
+/// using only column operations (no MPHF lookup per kmer).
+///
+/// `threshold` semantics follow [`MatrixGroupOps::partial_group_presence_count`]:
+/// a slot contributes to the count when its value is **≥ threshold**.
+/// To match the row-level filter (`value > t`), callers should pass `t + 1`.
+#[derive(Debug, Clone)]
+pub enum FilterMask {
+    /// Slot passes if count of columns in `indices` with value ≥ `threshold` is ≥ `min_count`.
+    PresenceGeq { indices: Vec<usize>, threshold: u32, min_count: usize },
+    /// Slot passes if count of columns in `indices` with value ≥ `threshold` is ≤ `max_count`.
+    PresenceLeq { indices: Vec<usize>, threshold: u32, max_count: usize },
+    /// Slot passes if sum of values across `indices` columns is ≥ `min_sum`.
+    SumGeq { indices: Vec<usize>, min_sum: u32 },
+    /// Slot passes if sum of values across `indices` columns is ≤ `max_sum`.
+    SumLeq { indices: Vec<usize>, max_sum: u32 },
+    /// Slot passes if it passes all sub-expressions. Empty `And` is always true.
+    And(Vec<FilterMask>),
+}
+
+/// Evaluate a [`FilterMask`] against `mat`, returning a per-slot `TempBitVec`
+/// where bit=1 means the slot passes the filter.
+pub fn eval_filter_mask(expr: &FilterMask, mat: &dyn MatrixGroupOps, n: usize) -> io::Result<TempBitVec> {
+    match expr {
+        FilterMask::PresenceGeq { indices, threshold, min_count } => {
+            let g = ColGroup::new("", indices.clone());
+            let counts = mat.partial_group_presence_count(&g, *threshold)?;
+            let mut b = TempBitVecBuilder::new(n)?;
+            let mc = *min_count as u32;
+            b.or_where(counts.view(), |v| v >= mc);
+            b.freeze()
+        }
+        FilterMask::PresenceLeq { indices, threshold, max_count } => {
+            let g = ColGroup::new("", indices.clone());
+            let counts = mat.partial_group_presence_count(&g, *threshold)?;
+            let mut b = TempBitVecBuilder::new(n)?;
+            let mc = *max_count as u32;
+            b.or_where(counts.view(), |v| v <= mc);
+            b.freeze()
+        }
+        FilterMask::SumGeq { indices, min_sum } => {
+            let g = ColGroup::new("", indices.clone());
+            let sums = mat.partial_group_sum(&g)?;
+            let mut b = TempBitVecBuilder::new(n)?;
+            let ms = *min_sum;
+            b.or_where(sums.view(), |v| v >= ms);
+            b.freeze()
+        }
+        FilterMask::SumLeq { indices, max_sum } => {
+            let g = ColGroup::new("", indices.clone());
+            let sums = mat.partial_group_sum(&g)?;
+            let mut b = TempBitVecBuilder::new(n)?;
+            let ms = *max_sum;
+            b.or_where(sums.view(), |v| v <= ms);
+            b.freeze()
+        }
+        FilterMask::And(parts) => {
+            let mut b = TempBitVecBuilder::new_ones(n)?;
+            for part in parts {
+                let m = eval_filter_mask(part, mat, n)?;
+                b.and(m.view());
+            }
+            b.freeze()
         }
     }
 }

src/obicompactvec/src/intmatrix.rs

@@ -1,7 +1,5 @@
-use std::cmp::Ordering;
-use std::collections::HashMap;
 use std::fs::{self, File};
-use std::io::{self, BufWriter, Write as _};
+use std::io::{self, BufWriter, Read as _, Write as _};
 use std::path::{Path, PathBuf};
 
 use memmap2::Mmap;
@@ -10,14 +8,13 @@ use rayon::prelude::*;
 
 use crate::bitmatrix::{pairwise_matrix, pairwise2_matrix};
 use crate::builder::PersistentCompactIntVecBuilder;
-use crate::colgroup::{ColGroup, MatrixGroupOps, inc_primary_bits};
-use crate::memoryintvec::MemoryIntVec;
-use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
-use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
-use crate::format::{byte_count_nonzero, byte_sum, HEADER_SIZE, OVERFLOW_ENTRY_SIZE, parse_index_entry, parse_overflow_entry};
+use crate::colgroup::{ColGroup, MatrixGroupOps};
+use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE};
 use crate::meta::MatrixMeta;
 use crate::reader::PersistentCompactIntVec;
-use crate::traits::{BitSliceMut, IntSlice, IntSliceMut};
+use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
+use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
+use crate::views::IntSliceView;
 
 fn col_path(dir: &Path, col: usize) -> PathBuf {
     dir.join(format!("col_{col:06}.pciv"))
@@ -48,9 +45,7 @@ impl ColumnarCompactIntMatrix {
     }
 
     pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
-        for (c, col) in self.cols.iter().enumerate() {
-            buf[c] = col.get(slot);
-        }
+        for (c, col) in self.cols.iter().enumerate() { buf[c] = col.get(slot); }
     }
 
     pub(crate) fn sum(&self) -> Array1<u64> {
@@ -72,31 +67,22 @@ impl ColumnarCompactIntMatrix {
     pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
         pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_bray_dist(self.col(j)))
     }
-
     pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
         pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_euclidean_dist(self.col(j)))
     }
-
-    pub(crate) fn partial_threshold_jaccard_dist_matrix(
-        &self, threshold: u32,
-    ) -> (Array2<u64>, Array2<u64>) {
-        pairwise2_matrix(self.n_cols(), |i, j| {
-            self.col(i).partial_threshold_jaccard_dist(self.col(j), threshold)
-        })
+    pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
+        pairwise2_matrix(self.n_cols(), |i, j| self.col(i).partial_threshold_jaccard_dist(self.col(j), threshold))
     }
-
     pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
         pairwise_matrix(self.n_cols(), |i, j| {
             self.col(i).partial_relfreq_bray_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
         })
     }
-
     pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
         pairwise_matrix(self.n_cols(), |i, j| {
             self.col(i).partial_relfreq_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
         })
     }
-
     pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
         pairwise_matrix(self.n_cols(), |i, j| {
             self.col(i).partial_hellinger_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
@@ -111,7 +97,6 @@ impl ColumnarCompactIntMatrix {
         meta.n_cols += 1;
         meta.save(dir)
     }
-
 }
 
 // ── PackedCompactIntMatrix ────────────────────────────────────────────────────
@@ -119,153 +104,12 @@ impl ColumnarCompactIntMatrix {
 const PCMX_MAGIC:  [u8; 4] = *b"PCMX";
 const PCMX_HEADER: usize   = 24; // magic(4) + pad(4) + n_rows(8) + n_cols(8)
 
-/// Per-column metadata pre-parsed from the embedded PCIV header.
 struct ColInfo {
-    primary_start: usize,  // absolute mmap offset to primary array
-    data_offset:   usize,  // absolute mmap offset to overflow array
+    primary_start: usize,
+    data_offset:   usize,
     n_overflow:    usize,
-    step:          usize,
-    index:         Vec<(usize, usize)>,
 }
 
-// ── PackedIntCol — lightweight column view backed by the shared mmap ──────────
-
-pub(crate) struct PackedIntCol<'a> {
-    primary:    &'a [u8],
-    overflow:   &'a [u8],  // raw bytes: n_overflow × OVERFLOW_ENTRY_SIZE
-    n_overflow: usize,
-    step:       usize,
-    index:      &'a [(usize, usize)],
-    n:          usize,
-}
-
-impl PackedIntCol<'_> {
-    fn overflow_get(&self, slot: usize) -> u32 {
-        let (pos_start, pos_end) = if self.step == 0 {
-            (0, self.n_overflow)
-        } else {
-            let i = self.index.partition_point(|&(s, _)| s <= slot).saturating_sub(1);
-            let start = self.index[i].1;
-            let end   = if i + 1 < self.index.len() { self.index[i + 1].1 } else { self.n_overflow };
-            (start, end)
-        };
-        let mut lo = pos_start;
-        let mut hi = pos_end;
-        while lo < hi {
-            let mid = lo + (hi - lo) / 2;
-            let (stored, val) = parse_overflow_entry(self.overflow, 0, mid);
-            match stored.cmp(&slot) {
-                Ordering::Equal   => return val,
-                Ordering::Less    => lo = mid + 1,
-                Ordering::Greater => hi = mid,
-            }
-        }
-        panic!("slot {slot} marked overflow but not found")
-    }
-}
-
-impl IntSlice for PackedIntCol<'_> {
-    fn len(&self) -> usize { self.n }
-
-    fn get(&self, slot: usize) -> u32 {
-        let v = self.primary[slot];
-        if v < 255 { v as u32 } else { self.overflow_get(slot) }
-    }
-
-    fn primary_bytes(&self) -> &[u8] { self.primary }
-
-    fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_ {
-        (0..self.n_overflow).map(|i| parse_overflow_entry(self.overflow, 0, i))
-    }
-
-    fn iter(&self) -> impl Iterator<Item = u32> + '_ {
-        PackedIntColIter {
-            primary:      self.primary,
-            overflow:     self.overflow,
-            slot:         0,
-            overflow_pos: 0,
-            n:            self.n,
-        }
-    }
-
-    fn sum(&self) -> u64 {
-        byte_sum(self.primary, (0..self.n_overflow).map(|i| parse_overflow_entry(self.overflow, 0, i).1))
-    }
-
-    fn count_nonzero(&self) -> u64 { byte_count_nonzero(self.primary) }
-}
-
-struct PackedIntColIter<'a> {
-    primary:      &'a [u8],
-    overflow:     &'a [u8],
-    slot:         usize,
-    overflow_pos: usize,
-    n:            usize,
-}
-
-impl Iterator for PackedIntColIter<'_> {
-    type Item = u32;
-
-    fn next(&mut self) -> Option<u32> {
-        if self.slot >= self.n { return None; }
-        let v = self.primary[self.slot];
-        self.slot += 1;
-        if v < 255 {
-            Some(v as u32)
-        } else {
-            let (_, val) = parse_overflow_entry(self.overflow, 0, self.overflow_pos);
-            self.overflow_pos += 1;
-            Some(val)
-        }
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        let rem = self.n - self.slot;
-        (rem, Some(rem))
-    }
-}
-
-impl ExactSizeIterator for PackedIntColIter<'_> {}
-
-// ── IntColView — uniform column access across Columnar and Packed ─────────────
-
-enum IntColViewInner<'a> {
-    Columnar(&'a PersistentCompactIntVec),
-    Packed(PackedIntCol<'a>),
-}
-
-/// Opaque column view returned by [`PersistentCompactIntMatrix::col_view`].
-/// Implements [`IntSlice`] uniformly for both Columnar and Packed matrix formats.
-pub struct IntColView<'a>(IntColViewInner<'a>);
-
-impl IntSlice for IntColView<'_> {
-    fn len(&self) -> usize {
-        match &self.0 { IntColViewInner::Columnar(c) => c.len(), IntColViewInner::Packed(c) => c.len() }
-    }
-    fn get(&self, slot: usize) -> u32 {
-        match &self.0 { IntColViewInner::Columnar(c) => c.get(slot), IntColViewInner::Packed(c) => c.get(slot) }
-    }
-    fn primary_bytes(&self) -> &[u8] {
-        match &self.0 { IntColViewInner::Columnar(c) => c.primary_bytes(), IntColViewInner::Packed(c) => c.primary_bytes() }
-    }
-    fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_ {
-        // Box<dyn Iterator> implements Iterator, satisfying RPITIT across two distinct types.
-        let it: Box<dyn Iterator<Item = (usize, u32)> + '_> = match &self.0 {
-            IntColViewInner::Columnar(c) => Box::new(c.overflow_entries()),
-            IntColViewInner::Packed(c)   => Box::new(c.overflow_entries()),
-        };
-        it
-    }
-    fn sum(&self) -> u64 {
-        match &self.0 { IntColViewInner::Columnar(c) => c.sum(), IntColViewInner::Packed(c) => c.sum() }
-    }
-    fn count_nonzero(&self) -> u64 {
-        match &self.0 { IntColViewInner::Columnar(c) => c.count_nonzero(), IntColViewInner::Packed(c) => c.count_nonzero() }
-    }
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-
 pub struct PackedCompactIntMatrix {
     mmap:    Mmap,
     n_rows:  usize,
@@ -289,52 +133,30 @@ impl PackedCompactIntMatrix {
         for c in 0..n_cols {
             let off_pos  = PCMX_HEADER + c * 8;
             let col_base = u64::from_le_bytes(mmap[off_pos..off_pos+8].try_into().unwrap()) as usize;
-            // Parse embedded PCIV header at col_base
             let n_ov   = u64::from_le_bytes(mmap[col_base+16..col_base+24].try_into().unwrap()) as usize;
-            let n_idx  = u64::from_le_bytes(mmap[col_base+24..col_base+32].try_into().unwrap()) as usize;
-            let step   = u64::from_le_bytes(mmap[col_base+32..col_base+40].try_into().unwrap()) as usize;
             let n_pciv = u64::from_le_bytes(mmap[col_base+8..col_base+16].try_into().unwrap())  as usize;
-
             let primary_start = col_base + HEADER_SIZE;
             let data_offset   = primary_start + n_pciv;
-            let index_offset  = data_offset + n_ov * OVERFLOW_ENTRY_SIZE;
-
-            let mut index = Vec::with_capacity(n_idx);
-            for i in 0..n_idx {
-                index.push(parse_index_entry(&mmap, index_offset, i));
-            }
-            columns.push(ColInfo { primary_start, data_offset, n_overflow: n_ov, step, index });
+            columns.push(ColInfo { primary_start, data_offset, n_overflow: n_ov });
         }
-
         Ok(Self { mmap, n_rows, n_cols, columns })
     }
 
-    pub(crate) fn col_slice(&self, c: usize) -> PackedIntCol<'_> {
+    pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> {
         let ci = &self.columns[c];
-        PackedIntCol {
-            primary:    &self.mmap[ci.primary_start..ci.primary_start + self.n_rows],
-            overflow:   &self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE],
-            n_overflow: ci.n_overflow,
-            step:       ci.step,
-            index:      &ci.index,
-            n:          self.n_rows,
-        }
+        let primary     = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
+        let overflow_raw = &self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE];
+        IntSliceView::new(primary, overflow_raw, ci.n_overflow, self.n_rows)
     }
 
     pub(crate) fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentCompactIntVecBuilder> {
-        let col = self.col_slice(c);
-        let overflow: HashMap<usize, u32> = col.overflow_entries().collect();
-        PersistentCompactIntVecBuilder::from_raw_primary(col.primary, overflow, path)
-    }
-
-    pub(crate) fn col_as_memory(&self, c: usize) -> MemoryIntVec {
-        MemoryIntVec::from(&self.col_slice(c))
+        let view = self.col_view(c);
+        let overflow: std::collections::HashMap<usize, u32> = view.overflow_entries().collect();
+        PersistentCompactIntVecBuilder::from_raw_primary(view.primary_bytes(), overflow, path)
     }
 
     #[inline]
-    pub(crate) fn get(&self, col: usize, slot: usize) -> u32 {
-        self.col_slice(col).get(slot)
-    }
+    pub(crate) fn get(&self, col: usize, slot: usize) -> u32 { self.col_view(col).get(slot) }
 
     pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
         for c in 0..self.n_cols { buf[c] = self.get(c, slot); }
@@ -346,121 +168,112 @@ impl PackedCompactIntMatrix {
 
     pub(crate) fn sum(&self) -> Array1<u64> {
         Array1::from_vec(
-            (0..self.n_cols).into_par_iter()
-                .map(|c| self.col_slice(c).sum())
-                .collect()
+            (0..self.n_cols).into_par_iter().map(|c| self.col_view(c).sum()).collect()
         )
     }
 
     pub(crate) fn count_nonzero(&self) -> Array1<u64> {
         Array1::from_vec(
-            (0..self.n_cols).into_par_iter()
-                .map(|c| self.col_slice(c).count_nonzero())
-                .collect()
+            (0..self.n_cols).into_par_iter().map(|c| self.col_view(c).count_nonzero()).collect()
         )
     }
 
-    // ── Pair primitives — sequential scan via col_slice().iter() ─────────────
-
     fn pair_partial_bray(&self, i: usize, j: usize) -> u64 {
-        self.col_slice(i).iter().zip(self.col_slice(j).iter())
-            .map(|(a, b)| a.min(b) as u64)
-            .sum()
+        self.col_view(i).iter().zip(self.col_view(j).iter()).map(|(a, b)| a.min(b) as u64).sum()
     }
-
     fn pair_partial_euclidean(&self, i: usize, j: usize) -> f64 {
-        self.col_slice(i).iter().zip(self.col_slice(j).iter())
-            .map(|(a, b)| { let d = a as f64 - b as f64; d * d })
-            .sum()
+        self.col_view(i).iter().zip(self.col_view(j).iter())
+            .map(|(a, b)| { let d = a as f64 - b as f64; d * d }).sum()
     }
-
     fn pair_partial_threshold_jaccard(&self, i: usize, j: usize, t: u32) -> (u64, u64) {
-        self.col_slice(i).iter().zip(self.col_slice(j).iter())
+        self.col_view(i).iter().zip(self.col_view(j).iter())
             .fold((0u64, 0u64), |(inter, uni), (a, b)| {
-                let ap = a >= t;
-                let bp = b >= t;
+                let ap = a >= t; let bp = b >= t;
                 (inter + (ap & bp) as u64, uni + (ap | bp) as u64)
             })
     }
-
     fn pair_partial_relfreq_bray(&self, i: usize, j: usize, si: f64, sj: f64) -> f64 {
         if si == 0.0 || sj == 0.0 { return 0.0; }
-        self.col_slice(i).iter().zip(self.col_slice(j).iter())
-            .map(|(a, b)| (a as f64 / si).min(b as f64 / sj))
-            .sum()
+        self.col_view(i).iter().zip(self.col_view(j).iter())
+            .map(|(a, b)| (a as f64 / si).min(b as f64 / sj)).sum()
     }
-
     fn pair_partial_relfreq_euclidean(&self, i: usize, j: usize, si: f64, sj: f64) -> f64 {
         if si == 0.0 || sj == 0.0 { return 0.0; }
-        self.col_slice(i).iter().zip(self.col_slice(j).iter())
-            .map(|(a, b)| { let d = a as f64 / si - b as f64 / sj; d * d })
-            .sum()
+        self.col_view(i).iter().zip(self.col_view(j).iter())
+            .map(|(a, b)| { let d = a as f64 / si - b as f64 / sj; d * d }).sum()
     }
-
     fn pair_partial_hellinger(&self, i: usize, j: usize, si: f64, sj: f64) -> f64 {
         if si == 0.0 || sj == 0.0 { return 0.0; }
-        self.col_slice(i).iter().zip(self.col_slice(j).iter())
-            .map(|(a, b)| { let d = (a as f64 / si).sqrt() - (b as f64 / sj).sqrt(); d * d })
-            .sum()
+        self.col_view(i).iter().zip(self.col_view(j).iter())
+            .map(|(a, b)| { let d = (a as f64 / si).sqrt() - (b as f64 / sj).sqrt(); d * d }).sum()
     }
 
-    // ── Matrix methods ────────────────────────────────────────────────────────
-
     pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
         pairwise_matrix(self.n_cols, |i, j| self.pair_partial_bray(i, j))
     }
-
     pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
         pairwise_matrix(self.n_cols, |i, j| self.pair_partial_euclidean(i, j))
     }
-
     pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, t: u32) -> (Array2<u64>, Array2<u64>) {
         pairwise2_matrix(self.n_cols, |i, j| self.pair_partial_threshold_jaccard(i, j, t))
     }
-
     pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
         pairwise_matrix(self.n_cols, |i, j| self.pair_partial_relfreq_bray(i, j, col_sums[i] as f64, col_sums[j] as f64))
     }
-
     pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
         pairwise_matrix(self.n_cols, |i, j| self.pair_partial_relfreq_euclidean(i, j, col_sums[i] as f64, col_sums[j] as f64))
     }
-
     pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
         pairwise_matrix(self.n_cols, |i, j| self.pair_partial_hellinger(i, j, col_sums[i] as f64, col_sums[j] as f64))
     }
 }
 
+/// Reads just the `n_cols` field from an existing packed matrix's header,
+/// without mapping the file. Used by `pack_compact_int_matrix` to tell a
+/// genuinely complete pack from a stale one that predates a later
+/// column-widening.
+fn packed_int_matrix_n_cols(path: &Path) -> io::Result<usize> {
+    let mut f = File::open(path)?;
+    let mut header = [0u8; PCMX_HEADER];
+    f.read_exact(&mut header)?;
+    Ok(u64::from_le_bytes(header[16..24].try_into().unwrap()) as usize)
+}
+
 /// Build `counts/matrix.pcmx` from existing `col_*.pciv` files.
 pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
     let packed_path = dir.join("matrix.pcmx");
-    if packed_path.exists() {
-        // Matrix complete; remove any leftover column files from a killed cleanup.
-        if let Ok(meta) = MatrixMeta::load(dir) {
-            for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
-            let _ = fs::remove_file(dir.join("meta.json"));
+
+    let meta = match MatrixMeta::load(dir) {
+        Ok(meta) => meta,
+        Err(e) => {
+            // No columnar data pending: either this layer was already
+            // packed and cleaned up (matrix.pcmx complete, nothing left to
+            // do), or genuinely nothing was ever written here.
+            return if packed_path.exists() { Ok(()) } else { Err(e) };
         }
+    };
+
+    // A `matrix.pcmx` can already exist here even though columnar data is
+    // still pending — e.g. copied verbatim from a merge's base source
+    // before this layer was widened with more genome columns (see
+    // `obikpartitionner::merge_partition`). Only skip (re-)packing if the
+    // existing file already reflects the current column count; otherwise
+    // the columnar files are newer and must be (re-)packed, overwriting the
+    // stale one — never silently discarded as "leftover cleanup".
+    if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
+        for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
+        let _ = fs::remove_file(dir.join("meta.json"));
         return Ok(());
     }
 
-    let meta = MatrixMeta::load(dir)?;
     let n_cols = meta.n_cols;
-
-    // Compute offsets from file sizes — no column data loaded into RAM.
     let col_sizes: Vec<u64> = (0..n_cols)
         .map(|c| fs::metadata(col_path(dir, c)).map(|m| m.len()))
         .collect::<io::Result<_>>()?;
-
     let header_size = (PCMX_HEADER + n_cols * 8) as u64;
     let mut col_offset = header_size;
     let mut offsets = Vec::with_capacity(n_cols);
-    for &size in &col_sizes {
-        offsets.push(col_offset);
-        col_offset += size;
-    }
-
-    // Write to a temp file; rename atomically so a killed process never leaves
-    // a truncated matrix.pcmx that would be mistaken for a complete file.
+    for &size in &col_sizes { offsets.push(col_offset); col_offset += size; }
     let tmp_path = dir.join("matrix.pcmx.tmp");
     let mut out = BufWriter::new(File::create(&tmp_path)?);
     out.write_all(&PCMX_MAGIC)?;
@@ -468,13 +281,10 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
     out.write_all(&(meta.n as u64).to_le_bytes())?;
     out.write_all(&(n_cols as u64).to_le_bytes())?;
     for &off in &offsets { out.write_all(&off.to_le_bytes())?; }
-    for c in 0..n_cols {
-        io::copy(&mut File::open(col_path(dir, c))?, &mut out)?;
-    }
+    for c in 0..n_cols { io::copy(&mut File::open(col_path(dir, c))?, &mut out)?; }
     out.flush()?;
     drop(out);
     fs::rename(&tmp_path, &packed_path)?;
-
     for c in 0..n_cols { fs::remove_file(col_path(dir, c))?; }
     fs::remove_file(dir.join("meta.json"))?;
     Ok(())
@@ -488,18 +298,14 @@ pub enum PersistentCompactIntMatrix {
 }
 
 impl PersistentCompactIntMatrix {
-    /// Open from `layer_dir`, auto-detecting Packed or Columnar.
     pub fn open(layer_dir: &Path) -> io::Result<Self> {
         let counts_dir = layer_dir.join("counts");
-
         if counts_dir.join("matrix.pcmx").exists() {
             return Ok(Self::Packed(PackedCompactIntMatrix::open(&counts_dir.join("matrix.pcmx"))?));
         }
-
         if MatrixMeta::load(&counts_dir).is_ok() {
             return Ok(Self::Columnar(ColumnarCompactIntMatrix::open(&counts_dir)?));
         }
-
         Err(io::Error::new(
             io::ErrorKind::NotFound,
             format!("no count matrix found in {} — run 'obikmer upgrade'", layer_dir.display()),
@@ -509,7 +315,6 @@ impl PersistentCompactIntMatrix {
     pub fn n(&self) -> usize {
         match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows }
     }
-
     pub fn n_cols(&self) -> usize {
         match self { Self::Columnar(m) => m.n_cols(), Self::Packed(m) => m.n_cols }
     }
@@ -521,10 +326,10 @@ impl PersistentCompactIntMatrix {
         }
     }
 
-    pub fn col_view(&self, c: usize) -> IntColView<'_> {
+    pub fn col_view(&self, c: usize) -> IntSliceView<'_> {
         match self {
-            Self::Columnar(m) => IntColView(IntColViewInner::Columnar(m.col(c))),
-            Self::Packed(m)   => IntColView(IntColViewInner::Packed(m.col_slice(c))),
+            Self::Columnar(m) => m.col(c).view(),
+            Self::Packed(m)   => m.col_view(c),
         }
     }
 
@@ -535,29 +340,18 @@ impl PersistentCompactIntMatrix {
         }
     }
 
-    pub fn col_as_memory(&self, c: usize) -> MemoryIntVec {
-        match self {
-            Self::Columnar(m) => MemoryIntVec::from(m.col(c)),
-            Self::Packed(m)   => m.col_as_memory(c),
-        }
-    }
-
     pub fn row(&self, slot: usize) -> Box<[u32]> {
         match self { Self::Columnar(m) => m.row(slot), Self::Packed(m) => m.row(slot) }
     }
-
     pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
         match self { Self::Columnar(m) => m.fill_row(slot, buf), Self::Packed(m) => m.fill_row(slot, buf) }
     }
-
     pub fn sum(&self) -> Array1<u64> {
         match self { Self::Columnar(m) => m.sum(), Self::Packed(m) => m.sum() }
     }
-
     pub fn count_nonzero(&self) -> Array1<u64> {
         match self { Self::Columnar(m) => m.count_nonzero(), Self::Packed(m) => m.count_nonzero() }
     }
-
     pub fn partial_bray_dist_matrix(&self) -> Array2<u64> {
         match self { Self::Columnar(m) => m.partial_bray_dist_matrix(), Self::Packed(m) => m.partial_bray_dist_matrix() }
     }
@@ -576,7 +370,6 @@ impl PersistentCompactIntMatrix {
     pub fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
         match self { Self::Columnar(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums), Self::Packed(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums) }
     }
-
     pub fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> {
         ColumnarCompactIntMatrix::append_column(dir, value_of)
     }
@@ -592,12 +385,12 @@ impl ColumnWeights for PersistentCompactIntMatrix {
 }
 
 impl CountPartials for PersistentCompactIntMatrix {
-    fn partial_bray(&self) -> Array2<u64>                        { self.partial_bray_dist_matrix() }
-    fn partial_euclidean(&self) -> Array2<f64>                   { self.partial_euclidean_dist_matrix() }
+    fn partial_bray(&self) -> Array2<u64>                                 { self.partial_bray_dist_matrix() }
+    fn partial_euclidean(&self) -> Array2<f64>                            { self.partial_euclidean_dist_matrix() }
     fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) { self.partial_threshold_jaccard_dist_matrix(t) }
-    fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64>     { self.partial_relfreq_bray_dist_matrix(g) }
-    fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_euclidean_dist_matrix(g) }
-    fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64>         { self.partial_hellinger_euclidean_dist_matrix(g) }
+    fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64>        { self.partial_relfreq_bray_dist_matrix(g) }
+    fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64>   { self.partial_relfreq_euclidean_dist_matrix(g) }
+    fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64>           { self.partial_hellinger_euclidean_dist_matrix(g) }
 }
 
 // ── Builder ───────────────────────────────────────────────────────────────────
@@ -613,16 +406,28 @@ impl PersistentCompactIntMatrixBuilder {
         fs::create_dir_all(dir)?;
         Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
     }
-
     pub fn n(&self)      -> usize { self.n }
     pub fn n_cols(&self) -> usize { self.n_cols }
-
     pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> {
         let path = col_path(&self.dir, self.n_cols);
         self.n_cols += 1;
         PersistentCompactIntVecBuilder::new(self.n, &path)
     }
 
+    pub fn add_col_from(&mut self, src: &TempCompactIntVec) -> io::Result<()> {
+        src.make_persistent(&col_path(&self.dir, self.n_cols))?;
+        self.n_cols += 1;
+        Ok(())
+    }
+
+    pub fn add_col_from_bit(&mut self, src: &TempBitVec) -> io::Result<()> {
+        let path = col_path(&self.dir, self.n_cols);
+        self.n_cols += 1;
+        let mut b = PersistentCompactIntVecBuilder::new(self.n, &path)?;
+        b.inc_present(src.view());
+        b.close()
+    }
+
     pub fn close(self) -> io::Result<()> {
         MatrixMeta { n: self.n, n_cols: self.n_cols }.save(&self.dir)
     }
@@ -634,30 +439,20 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
     fn partial_group_presence_count(&self, g: &ColGroup, threshold: u32) -> io::Result<TempCompactIntVec> {
         let n = self.n();
         if g.indices.len() < 255 {
-            // Fast path: counts fit in u8 — accumulate directly into raw bytes.
             let mut builder = TempCompactIntVecBuilder::new(n)?;
-            {
-                let primary = builder.primary_bytes_mut();
-                for &c in &g.indices {
-                    let mask = self.col_view(c).cmp_scalar(|v| v >= threshold);
-                    inc_primary_bits(primary, &mask);
-                }
+            for &c in &g.indices {
+                builder.inc_predicate_fast(self.col_view(c), |v| v >= threshold);
             }
             builder.freeze()
         } else {
-            // Slow path: chunk by 254 to keep per-chunk u8 safe, then add chunks.
             let mut result = TempCompactIntVecBuilder::new(n)?;
             for chunk in g.indices.chunks(254) {
-                let mut chunk_builder = TempCompactIntVecBuilder::new(n)?;
-                {
-                    let primary = chunk_builder.primary_bytes_mut();
-                    for &c in chunk {
-                        let mask = self.col_view(c).cmp_scalar(|v| v >= threshold);
-                        inc_primary_bits(primary, &mask);
-                    }
+                let mut chunk_b = TempCompactIntVecBuilder::new(n)?;
+                for &c in chunk {
+                    chunk_b.inc_predicate_fast(self.col_view(c), |v| v >= threshold);
                 }
-                let chunk_frozen = chunk_builder.freeze()?;
-                IntSliceMut::add(&mut result, &chunk_frozen);
+                let frozen = chunk_b.freeze()?;
+                result.add(frozen.view());
             }
             result.freeze()
         }
@@ -666,10 +461,7 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
     fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
         let n = self.n();
         let mut result = TempCompactIntVecBuilder::new(n)?;
-        for &c in &g.indices {
-            let view = self.col_view(c);
-            IntSliceMut::add(&mut result, &view);
-        }
+        for &c in &g.indices { result.add(self.col_view(c)); }
         result.freeze()
     }
 
@@ -677,9 +469,25 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
         let n = self.n();
         let mut result = TempBitVecBuilder::new(n)?;
         for &c in &g.indices {
-            let mask = self.col_view(c).cmp_scalar(|v| v >= threshold);
-            result.or(&mask);
+            result.or_where(self.col_view(c), |v| v >= threshold);
         }
         result.freeze()
     }
+
+    fn partial_group_min(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
+        let n = self.n();
+        let mut result = TempCompactIntVecBuilder::new(n)?;
+        if let Some((&first, rest)) = g.indices.split_first() {
+            result.add(self.col_view(first));
+            for &c in rest { result.min(self.col_view(c)); }
+        }
+        result.freeze()
+    }
+
+    fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
+        let n = self.n();
+        let mut result = TempCompactIntVecBuilder::new(n)?;
+        for &c in &g.indices { result.max(self.col_view(c)); }
+        result.freeze()
+    }
 }

src/obicompactvec/src/lib.rs

@@ -5,26 +5,24 @@ mod colgroup;
 mod format;
 mod intmatrix;
 mod layer_meta;
-mod memoryintvec;
-mod memoryvec;
 mod meta;
 mod reader;
 mod tempbitvec;
 mod tempintvec;
+mod views;
 pub mod traits;
 
 pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
-pub use bitmatrix::{BitColView, PersistentBitMatrix, PersistentBitMatrixBuilder, pack_bit_matrix};
+pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, pack_bit_matrix};
 pub use builder::PersistentCompactIntVecBuilder;
-pub use colgroup::{ColGroup, MatrixGroupOps};
-pub use intmatrix::{IntColView, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
+pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
+pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
 pub use layer_meta::LayerMeta;
-pub use memoryintvec::{MemoryIntIter, MemoryIntVec};
-pub use memoryvec::MemoryBitVec;
-pub use reader::PersistentCompactIntVec;
-pub use tempbitvec::TempBitVec;
-pub use tempintvec::TempCompactIntVec;
-pub use traits::{BitPartials, BitSlice, BitSliceMut, BitToInt, ColumnWeights, CountPartials, IntSlice, IntSliceMut, IntToBit};
+pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter};
+pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
+pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
+pub use traits::{BitPartials, ColumnWeights, CountPartials};
+pub use views::{BitSliceView, BitSliceIter, IntSliceView, IntSliceViewIter};
 
 #[cfg(test)]
 #[path = "tests/mod.rs"]

src/obicompactvec/src/memoryintvec.rs

@@ -1,186 +0,0 @@
-use std::collections::HashMap;
-use std::io;
-use std::ops::{Add, AddAssign, Sub, SubAssign};
-use std::path::Path;
-
-use crate::builder::PersistentCompactIntVecBuilder;
-use crate::format::{byte_count_nonzero, byte_sum};
-use crate::traits::{IntSlice, IntSliceMut};
-
-// ── MemoryIntVec ──────────────────────────────────────────────────────────────
-
-#[derive(Clone)]
-pub struct MemoryIntVec {
-    primary:  Vec<u8>,
-    overflow: HashMap<usize, u32>,
-    n:        usize,
-}
-
-impl MemoryIntVec {
-    pub fn new(n: usize) -> Self {
-        Self { primary: vec![0u8; n], overflow: HashMap::new(), n }
-    }
-
-    pub fn len(&self) -> usize { self.n }
-    pub fn is_empty(&self) -> bool { self.n == 0 }
-
-    /// Construct directly from a pre-built primary array (no overflow — all values < 255).
-    pub(crate) fn from_primary(primary: Vec<u8>) -> Self {
-        let n = primary.len();
-        Self { primary, overflow: HashMap::new(), n }
-    }
-
-    pub(crate) fn from_primary_and_overflow(primary: Vec<u8>, overflow: HashMap<usize, u32>) -> Self {
-        let n = primary.len();
-        Self { primary, overflow, n }
-    }
-
-    pub(crate) fn primary_bytes(&self) -> &[u8] { &self.primary }
-    pub(crate) fn overflow_map(&self) -> &HashMap<usize, u32> { &self.overflow }
-
-    pub fn get(&self, slot: usize) -> u32 {
-        match self.primary[slot] {
-            255 => *self.overflow.get(&slot).expect("sentinel without overflow entry"),
-            v   => v as u32,
-        }
-    }
-
-    pub fn sum(&self) -> u64 {
-        byte_sum(&self.primary, self.overflow.values().copied())
-    }
-
-    pub fn count_nonzero(&self) -> u64 {
-        byte_count_nonzero(&self.primary)
-    }
-
-    pub fn filled(n: usize, value: u32) -> Self {
-        if value < 255 {
-            Self { primary: vec![value as u8; n], overflow: HashMap::new(), n }
-        } else {
-            Self { primary: vec![255u8; n], overflow: (0..n).map(|i| (i, value)).collect(), n }
-        }
-    }
-
-    pub fn iter(&self) -> MemoryIntIter<'_> {
-        MemoryIntIter { vec: self, slot: 0 }
-    }
-
-    /// Write to disk and return a writable builder at `path`.
-    pub fn persist(&self, path: &Path) -> io::Result<PersistentCompactIntVecBuilder> {
-        PersistentCompactIntVecBuilder::from_memory(self, path)
-    }
-}
-
-// ── IntSlice / IntSliceMut ────────────────────────────────────────────────────
-
-impl IntSlice for MemoryIntVec {
-    fn len(&self) -> usize { self.n }
-    fn get(&self, slot: usize) -> u32 { self.get(slot) }
-    fn primary_bytes(&self) -> &[u8] { &self.primary }
-    fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_ {
-        self.overflow.iter().map(|(&k, &v)| (k, v))
-    }
-    fn iter(&self) -> impl Iterator<Item = u32> + '_ { self.iter() }
-    fn sum(&self) -> u64 { self.sum() }
-    fn count_nonzero(&self) -> u64 { self.count_nonzero() }
-}
-
-impl IntSliceMut for MemoryIntVec {
-    fn set(&mut self, slot: usize, value: u32) {
-        if value < 255 {
-            self.primary[slot] = value as u8;
-            self.overflow.remove(&slot);
-        } else {
-            self.primary[slot] = 255;
-            self.overflow.insert(slot, value);
-        }
-    }
-    fn primary_bytes_mut(&mut self) -> &mut [u8] { &mut self.primary }
-    fn clear_overflow(&mut self) { self.overflow.clear(); }
-}
-
-// ── From conversions ──────────────────────────────────────────────────────────
-
-impl MemoryIntVec {
-    /// Bulk copy from another `MemoryIntVec`: memcpy for the primary bytes,
-    /// clone for the overflow map.
-    pub fn copy_from_memory(&mut self, src: &MemoryIntVec) {
-        assert_eq!(self.n, src.n, "MemoryIntVec length mismatch");
-        self.primary.copy_from_slice(&src.primary);
-        self.overflow = src.overflow.clone();
-    }
-}
-
-impl<S: IntSlice> From<&S> for MemoryIntVec {
-    fn from(src: &S) -> Self {
-        Self::from_primary_and_overflow(
-            src.primary_bytes().to_vec(),
-            src.overflow_entries().collect(),
-        )
-    }
-}
-
-// ── std::ops — owned (consumes lhs) ──────────────────────────────────────────
-
-impl<B: IntSlice> Add<&B> for MemoryIntVec {
-    type Output = MemoryIntVec;
-    fn add(mut self, rhs: &B) -> MemoryIntVec { IntSliceMut::add(&mut self, rhs); self }
-}
-
-impl<B: IntSlice> Sub<&B> for MemoryIntVec {
-    type Output = MemoryIntVec;
-    fn sub(mut self, rhs: &B) -> MemoryIntVec { self.diff(rhs); self }
-}
-
-// ── std::ops — borrowed (clones lhs) ─────────────────────────────────────────
-
-impl<B: IntSlice> Add<&B> for &MemoryIntVec {
-    type Output = MemoryIntVec;
-    fn add(self, rhs: &B) -> MemoryIntVec { self.clone().add(rhs) }
-}
-
-impl<B: IntSlice> Sub<&B> for &MemoryIntVec {
-    type Output = MemoryIntVec;
-    fn sub(self, rhs: &B) -> MemoryIntVec { self.clone().sub(rhs) }
-}
-
-// ── std::ops — in-place assign ────────────────────────────────────────────────
-
-impl<B: IntSlice> AddAssign<&B> for MemoryIntVec {
-    fn add_assign(&mut self, rhs: &B) { IntSliceMut::add(self, rhs); }
-}
-
-impl<B: IntSlice> SubAssign<&B> for MemoryIntVec {
-    fn sub_assign(&mut self, rhs: &B) { self.diff(rhs); }
-}
-
-// ── Iterator ──────────────────────────────────────────────────────────────────
-
-pub struct MemoryIntIter<'a> {
-    vec: &'a MemoryIntVec,
-    slot: usize,
-}
-
-impl Iterator for MemoryIntIter<'_> {
-    type Item = u32;
-
-    fn next(&mut self) -> Option<u32> {
-        if self.slot >= self.vec.n { return None; }
-        let v = self.vec.get(self.slot);
-        self.slot += 1;
-        Some(v)
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        let rem = self.vec.n - self.slot;
-        (rem, Some(rem))
-    }
-}
-
-impl ExactSizeIterator for MemoryIntIter<'_> {}
-
-impl<'a> IntoIterator for &'a MemoryIntVec {
-    type Item = u32;
-    type IntoIter = MemoryIntIter<'a>;
-    fn into_iter(self) -> MemoryIntIter<'a> { self.iter() }
-}

src/obicompactvec/src/memoryvec.rs

@@ -1,138 +0,0 @@
-use std::io;
-use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not};
-use std::path::Path;
-
-use crate::bitvec::{BitIter, PersistentBitVecBuilder, n_words};
-use crate::traits::{BitSlice, BitSliceMut};
-
-// ── MemoryBitVec ──────────────────────────────────────────────────────────────
-
-#[derive(Clone)]
-pub struct MemoryBitVec {
-    words: Vec<u64>,
-    n: usize,
-}
-
-impl MemoryBitVec {
-    pub fn new(n: usize) -> Self {
-        Self { words: vec![0u64; n_words(n)], n }
-    }
-
-    pub fn ones(n: usize) -> Self {
-        let rem = n % 64;
-        let mut words = vec![u64::MAX; n_words(n)];
-        if rem != 0 {
-            if let Some(last) = words.last_mut() { *last = (1u64 << rem) - 1; }
-        }
-        Self { words, n }
-    }
-
-    pub(crate) fn from_words(words: Vec<u64>, n: usize) -> Self {
-        Self { words, n }
-    }
-
-    pub fn len(&self) -> usize { self.n }
-    pub fn is_empty(&self) -> bool { self.n == 0 }
-
-    pub fn get(&self, slot: usize) -> bool {
-        (self.words[slot >> 6] >> (slot & 63)) & 1 != 0
-    }
-
-    /// Write to disk and return a writable builder positioned at the same path.
-    pub fn persist(&self, path: &Path) -> io::Result<PersistentBitVecBuilder> {
-        let mut b = PersistentBitVecBuilder::new(self.n, path)?;
-        b.copy_from(self);
-        Ok(b)
-    }
-}
-
-// ── BitSlice / BitSliceMut ────────────────────────────────────────────────────
-
-impl BitSlice for MemoryBitVec {
-    fn len(&self) -> usize { self.n }
-    fn words(&self) -> &[u64] { &self.words }
-}
-
-impl BitSliceMut for MemoryBitVec {
-    fn words_mut(&mut self) -> &mut [u64] { &mut self.words }
-}
-
-// ── From conversions ──────────────────────────────────────────────────────────
-
-impl<S: BitSlice> From<&S> for MemoryBitVec {
-    fn from(src: &S) -> Self {
-        Self { words: src.words().to_vec(), n: src.len() }
-    }
-}
-
-// ── std::ops — owned (consumes lhs) ──────────────────────────────────────────
-
-impl<B: BitSlice> BitAnd<&B> for MemoryBitVec {
-    type Output = MemoryBitVec;
-    fn bitand(mut self, rhs: &B) -> MemoryBitVec { self.and(rhs); self }
-}
-
-impl<B: BitSlice> BitOr<&B> for MemoryBitVec {
-    type Output = MemoryBitVec;
-    fn bitor(mut self, rhs: &B) -> MemoryBitVec { self.or(rhs); self }
-}
-
-impl<B: BitSlice> BitXor<&B> for MemoryBitVec {
-    type Output = MemoryBitVec;
-    fn bitxor(mut self, rhs: &B) -> MemoryBitVec { self.xor(rhs); self }
-}
-
-impl Not for MemoryBitVec {
-    type Output = MemoryBitVec;
-    fn not(mut self) -> MemoryBitVec { BitSliceMut::not(&mut self); self }
-}
-
-// ── std::ops — borrowed (clones lhs) ─────────────────────────────────────────
-
-impl<B: BitSlice> BitAnd<&B> for &MemoryBitVec {
-    type Output = MemoryBitVec;
-    fn bitand(self, rhs: &B) -> MemoryBitVec { self.clone().bitand(rhs) }
-}
-
-impl<B: BitSlice> BitOr<&B> for &MemoryBitVec {
-    type Output = MemoryBitVec;
-    fn bitor(self, rhs: &B) -> MemoryBitVec { self.clone().bitor(rhs) }
-}
-
-impl<B: BitSlice> BitXor<&B> for &MemoryBitVec {
-    type Output = MemoryBitVec;
-    fn bitxor(self, rhs: &B) -> MemoryBitVec { self.clone().bitxor(rhs) }
-}
-
-impl Not for &MemoryBitVec {
-    type Output = MemoryBitVec;
-    fn not(self) -> MemoryBitVec { !self.clone() }
-}
-
-// ── std::ops — in-place assign ────────────────────────────────────────────────
-
-impl<B: BitSlice> BitAndAssign<&B> for MemoryBitVec {
-    fn bitand_assign(&mut self, rhs: &B) { self.and(rhs); }
-}
-
-impl<B: BitSlice> BitOrAssign<&B> for MemoryBitVec {
-    fn bitor_assign(&mut self, rhs: &B) { self.or(rhs); }
-}
-
-impl<B: BitSlice> BitXorAssign<&B> for MemoryBitVec {
-    fn bitxor_assign(&mut self, rhs: &B) { self.xor(rhs); }
-}
-
-// ── Iterator ──────────────────────────────────────────────────────────────────
-
-impl MemoryBitVec {
-    pub fn iter(&self) -> BitIter<'_> {
-        BitIter { words: &self.words, slot: 0, n: self.n }
-    }
-}
-
-impl<'a> IntoIterator for &'a MemoryBitVec {
-    type Item = bool;
-    type IntoIter = BitIter<'a>;
-    fn into_iter(self) -> BitIter<'a> { self.iter() }
-}

src/obicompactvec/src/reader.rs

@@ -5,6 +5,7 @@ use std::path::{Path, PathBuf};
 use memmap2::Mmap;
 
 use crate::format::{byte_count_nonzero, byte_sum, HEADER_SIZE, MAGIC, OVERFLOW_ENTRY_SIZE, parse_index_entry};
+use crate::views::IntSliceView;
 
 pub struct PersistentCompactIntVec {
     mmap: Mmap,
@@ -18,97 +19,60 @@ pub struct PersistentCompactIntVec {
 }
 
 impl PersistentCompactIntVec {
-    /// Opens a persistent compact int vector from the given path.
     pub fn open(path: &Path) -> io::Result<Self> {
         let mmap = unsafe { Mmap::map(&File::open(path)?)? };
 
         if mmap.len() < HEADER_SIZE {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidData,
-                "PCIV file too short",
-            ));
+            return Err(io::Error::new(io::ErrorKind::InvalidData, "PCIV file too short"));
         }
         if &mmap[0..4] != &MAGIC {
             return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PCIV magic"));
         }
 
-        let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
+        let n          = u64::from_le_bytes(mmap[8..16].try_into().unwrap())  as usize;
         let n_overflow = u64::from_le_bytes(mmap[16..24].try_into().unwrap()) as usize;
-        let n_index = u64::from_le_bytes(mmap[24..32].try_into().unwrap()) as usize;
-        let step = u64::from_le_bytes(mmap[32..40].try_into().unwrap()) as usize;
+        let n_index    = u64::from_le_bytes(mmap[24..32].try_into().unwrap()) as usize;
+        let step       = u64::from_le_bytes(mmap[32..40].try_into().unwrap()) as usize;
 
         let primary_offset = HEADER_SIZE;
-        let data_offset = primary_offset + n;
-        let index_offset = data_offset + n_overflow * OVERFLOW_ENTRY_SIZE;
+        let data_offset    = primary_offset + n;
+        let index_offset   = data_offset + n_overflow * OVERFLOW_ENTRY_SIZE;
 
         let mut index = Vec::with_capacity(n_index);
         for i in 0..n_index {
             index.push(parse_index_entry(&mmap, index_offset, i));
         }
 
-        Ok(Self {
-            mmap,
-            n,
-            n_overflow,
-            step,
-            index,
-            primary_offset,
-            data_offset,
-            path: path.to_path_buf(),
-        })
+        Ok(Self { mmap, n, n_overflow, step, index, primary_offset, data_offset, path: path.to_path_buf() })
     }
 
-    /// Returns the path of the compact int vector file.
-    pub fn path(&self) -> &Path {
-        &self.path
-    }
+    pub fn path(&self) -> &Path { &self.path }
+    pub fn len(&self)      -> usize { self.n }
+    pub fn is_empty(&self) -> bool  { self.n == 0 }
 
-    /// Returns the length of the compact int vector.
-    pub fn len(&self) -> usize {
-        self.n
-    }
-
-    /// Returns whether the compact int vector is empty.
-    pub fn is_empty(&self) -> bool {
-        self.n == 0
-    }
-
-    /// Returns the value at the given slot.
     pub fn get(&self, slot: usize) -> u32 {
         match self.mmap[self.primary_offset + slot] {
             255 => self.overflow_get(slot),
-            v => v as u32,
+            v   => v as u32,
         }
     }
 
-    /// Returns the value at the given slot from the overflow region.
     fn overflow_get(&self, slot: usize) -> u32 {
-        let pos_start;
-        let pos_end;
-
-        if self.step == 0 {
-            pos_start = 0;
-            pos_end = self.n_overflow;
+        let (pos_start, pos_end) = if self.step == 0 {
+            (0, self.n_overflow)
         } else {
-            let i = self
-                .index
-                .partition_point(|&(s, _)| s <= slot)
-                .saturating_sub(1);
-            pos_start = self.index[i].1;
-            pos_end = if i + 1 < self.index.len() {
-                self.index[i + 1].1
-            } else {
-                self.n_overflow
-            };
-        }
-
+            let i = self.index.partition_point(|&(s, _)| s <= slot).saturating_sub(1);
+            let start = self.index[i].1;
+            let end = if i + 1 < self.index.len() { self.index[i + 1].1 } else { self.n_overflow };
+            (start, end)
+        };
         let mut lo = pos_start;
         let mut hi = pos_end;
         while lo < hi {
             let mid = lo + (hi - lo) / 2;
             match self.data_slot(mid).cmp(&slot) {
-                std::cmp::Ordering::Equal => return self.data_value(mid),
-                std::cmp::Ordering::Less => lo = mid + 1,
+                std::cmp::Ordering::Equal   => return self.data_value(mid),
+                std::cmp::Ordering::Less    => lo = mid + 1,
                 std::cmp::Ordering::Greater => hi = mid,
             }
         }
@@ -116,14 +80,12 @@ impl PersistentCompactIntVec {
     }
 
     #[inline]
-    /// Returns the slot at the given index in the overflow region.
     fn data_slot(&self, i: usize) -> usize {
         let off = self.data_offset + i * OVERFLOW_ENTRY_SIZE;
         u64::from_le_bytes(self.mmap[off..off + 8].try_into().unwrap()) as usize
     }
 
     #[inline]
-    /// Returns the value at the given index in the overflow region.
     fn data_value(&self, i: usize) -> u32 {
         let off = self.data_offset + i * OVERFLOW_ENTRY_SIZE + 8;
         u32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap())
@@ -139,121 +101,70 @@ impl PersistentCompactIntVec {
         byte_count_nonzero(primary)
     }
 
-    #[inline]
-    /// Returns the Bray-Curtis distance between two compact int vectors.
+    /// Lightweight zero-copy view — primary and overflow point into the mmap.
+    pub fn view(&self) -> IntSliceView<'_> {
+        let primary = &self.mmap[self.primary_offset..self.primary_offset + self.n];
+        let overflow_raw = &self.mmap[self.data_offset..self.data_offset + self.n_overflow * OVERFLOW_ENTRY_SIZE];
+        IntSliceView::new(primary, overflow_raw, self.n_overflow, self.n)
+    }
+
+    pub fn iter(&self) -> Iter<'_> {
+        Iter { pciv: self, slot: 0, overflow_pos: 0 }
+    }
+
+    // ── Distance methods ──────────────────────────────────────────────────────
+
     pub fn bray_dist(&self, other: &PersistentCompactIntVec) -> f64 {
         let sum_min = self.partial_bray_dist(other);
         let denom = self.sum() + other.sum();
-        if denom == 0 {
-            return 0.0;
-        }
-        1.0 - 2.0 * sum_min as f64 / denom as f64
+        if denom == 0 { 0.0 } else { 1.0 - 2.0 * sum_min as f64 / denom as f64 }
     }
 
-    /// Returns `Σ_slot min(self[slot], other[slot])` — the additive numerator of Bray-Curtis.
-    /// The denominator `sum_a + sum_b` is obtained from `self.sum() + other.sum()`.
     pub fn partial_bray_dist(&self, other: &PersistentCompactIntVec) -> u64 {
         assert_eq!(self.n, other.len(), "length mismatch");
-        self.iter()
-            .zip(other.iter())
-            .map(|(a, b)| a.min(b) as u64)
-            .sum()
+        self.iter().zip(other.iter()).map(|(a, b)| a.min(b) as u64).sum()
     }
 
-    /// Returns the relative frequency Bray-Curtis distance between two compact int vectors.
-    ///
-    /// This is a variant of [`bray_dist`] that uses relative frequencies instead of raw counts.
     pub fn relfreq_bray_dist(&self, other: &PersistentCompactIntVec) -> f64 {
         assert_eq!(self.n, other.len(), "length mismatch");
-        let sum_a = self.sum() as f64;
-        let sum_b = other.sum() as f64;
-        if sum_a == 0.0 && sum_b == 0.0 {
-            return 0.0;
-        }
-        let sum_min = self.partial_relfreq_bray_dist(other, sum_a, sum_b);
-        1.0 - sum_min
+        let sa = self.sum() as f64;
+        let sb = other.sum() as f64;
+        if sa == 0.0 && sb == 0.0 { return 0.0; }
+        1.0 - self.partial_relfreq_bray_dist(other, sa, sb)
     }
 
-    /// Returns the partial relative frequency Bray-Curtis distance between two compact int vectors.
-    ///
-    /// This is used internally by [`relfreq_bray_dist`] and to easily compute the relative frequency
-    /// Bray-Curtis distance over a set of vector pairs.
-    ///
-    /// Arguments:
-    /// - `other`: the other compact int vector to compare with
-    /// - `sum_a`: the sum of the first vector's counts
-    /// - `sum_b`: the sum of the second vector's counts
-    ///
-    /// Returns the sum of the minimum relative frequencies at each index.
-    pub fn partial_relfreq_bray_dist(
-        &self,
-        other: &PersistentCompactIntVec,
-        sum_a: f64,
-        sum_b: f64,
-    ) -> f64 {
+    pub fn partial_relfreq_bray_dist(&self, other: &PersistentCompactIntVec, sum_a: f64, sum_b: f64) -> f64 {
         assert_eq!(self.n, other.len(), "length mismatch");
-        let sum_min: f64 = self
-            .iter()
-            .zip(other.iter())
+        self.iter().zip(other.iter())
             .map(|(a, b)| {
                 let pa = if sum_a > 0.0 { a as f64 / sum_a } else { 0.0 };
                 let pb = if sum_b > 0.0 { b as f64 / sum_b } else { 0.0 };
                 pa.min(pb)
             })
-            .sum();
-        sum_min
+            .sum()
     }
 
-    /// Returns the euclidean distance between two compact int vectors.
     pub fn euclidean_dist(&self, other: &PersistentCompactIntVec) -> f64 {
         self.partial_euclidean_dist(other).sqrt()
     }
 
-    /// Returns the partial euclidean distance between two compact int vectors.
-    ///
-    /// This is used internally by [`euclidean_dist`] and to easily compute the euclidean distance
-    /// over a set of vector pairs.
-    ///
-    /// The result is the sum of the squared differences between corresponding elements of the two
-    /// vectors.
     pub fn partial_euclidean_dist(&self, other: &PersistentCompactIntVec) -> f64 {
         assert_eq!(self.n, other.len(), "length mismatch");
-        self.iter()
-            .zip(other.iter())
-            .map(|(a, b)| {
-                let d = a as f64 - b as f64;
-                d * d
-            })
+        self.iter().zip(other.iter())
+            .map(|(a, b)| { let d = a as f64 - b as f64; d * d })
             .sum()
     }
 
-    /// Returns the relative frequency euclidean distance between two compact int vectors.
-    ///
-    /// This is a variant of [`euclidean_dist`] that uses relative frequencies instead of raw counts.
     pub fn relfreq_euclidean_dist(&self, other: &PersistentCompactIntVec) -> f64 {
-        assert_eq!(self.n, other.len(), "length mismatch");
-        let sum_a = self.sum() as f64;
-        let sum_b = other.sum() as f64;
-        if sum_a == 0.0 && sum_b == 0.0 {
-            return 0.0;
-        }
-        self.partial_relfreq_euclidean_dist(other, sum_a, sum_b)
-            .sqrt()
+        let sa = self.sum() as f64;
+        let sb = other.sum() as f64;
+        if sa == 0.0 && sb == 0.0 { return 0.0; }
+        self.partial_relfreq_euclidean_dist(other, sa, sb).sqrt()
     }
 
-    /// Returns the partial relative frequency euclidean distance between two compact int vectors.
-    ///
-    /// This is used internally by [`relfreq_euclidean_dist`] and to easily compute the relative frequency
-    /// euclidean distance over a set of vector pairs.
-    pub fn partial_relfreq_euclidean_dist(
-        &self,
-        other: &PersistentCompactIntVec,
-        sum_a: f64,
-        sum_b: f64,
-    ) -> f64 {
+    pub fn partial_relfreq_euclidean_dist(&self, other: &PersistentCompactIntVec, sum_a: f64, sum_b: f64) -> f64 {
         assert_eq!(self.n, other.len(), "length mismatch");
-        self.iter()
-            .zip(other.iter())
+        self.iter().zip(other.iter())
             .map(|(a, b)| {
                 let pa = if sum_a > 0.0 { a as f64 / sum_a } else { 0.0 };
                 let pb = if sum_b > 0.0 { b as f64 / sum_b } else { 0.0 };
@@ -263,46 +174,19 @@ impl PersistentCompactIntVec {
             .sum()
     }
 
-    /// Returns the Euclidean distance between two compact int vectors using the Hellinger transform.
-    ///
-    /// The Hellinger transform is applied to the raw counts of each vector, and the result is
-    /// the Euclidean distance between the transformed vectors. The Hellinger transform is defined
-    /// as the square root of the relative frequencies.
     pub fn hellinger_euclidean_dist(&self, other: &PersistentCompactIntVec) -> f64 {
-        assert_eq!(self.n, other.len(), "length mismatch");
-        let sum_a = self.sum() as f64;
-        let sum_b = other.sum() as f64;
-        if sum_a == 0.0 && sum_b == 0.0 {
-            return 0.0;
-        }
-        self.partial_hellinger_euclidean_dist(other, sum_a, sum_b)
-            .sqrt()
+        let sa = self.sum() as f64;
+        let sb = other.sum() as f64;
+        if sa == 0.0 && sb == 0.0 { return 0.0; }
+        self.partial_hellinger_euclidean_dist(other, sa, sb).sqrt()
     }
 
-    /// Returns the partial Hellinger Euclidean distance between two compact int vectors.
-    ///
-    /// This is used internally by [`hellinger_euclidean_dist`] and to easily compute the Hellinger
-    /// Euclidean distance over a set of vector pairs.
-    pub fn partial_hellinger_euclidean_dist(
-        &self,
-        other: &PersistentCompactIntVec,
-        sum_a: f64,
-        sum_b: f64,
-    ) -> f64 {
+    pub fn partial_hellinger_euclidean_dist(&self, other: &PersistentCompactIntVec, sum_a: f64, sum_b: f64) -> f64 {
         assert_eq!(self.n, other.len(), "length mismatch");
-        self.iter()
-            .zip(other.iter())
+        self.iter().zip(other.iter())
             .map(|(a, b)| {
-                let pa = if sum_a > 0.0 {
-                    (a as f64 / sum_a).sqrt()
-                } else {
-                    0.0
-                };
-                let pb = if sum_b > 0.0 {
-                    (b as f64 / sum_b).sqrt()
-                } else {
-                    0.0
-                };
+                let pa = if sum_a > 0.0 { (a as f64 / sum_a).sqrt() } else { 0.0 };
+                let pb = if sum_b > 0.0 { (b as f64 / sum_b).sqrt() } else { 0.0 };
                 let d = pa - pb;
                 d * d
             })
@@ -314,22 +198,13 @@ impl PersistentCompactIntVec {
     }
 
     pub fn threshold_jaccard_dist(&self, other: &PersistentCompactIntVec, threshold: u32) -> f64 {
-        assert_eq!(self.n, other.len(), "length mismatch");
         let (intersection, union) = self.partial_threshold_jaccard_dist(other, threshold);
-        if union == 0 {
-            return 0.0;
-        }
-        1.0 - intersection as f64 / union as f64
+        if union == 0 { 0.0 } else { 1.0 - intersection as f64 / union as f64 }
     }
 
-    pub fn partial_threshold_jaccard_dist(
-        &self,
-        other: &PersistentCompactIntVec,
-        threshold: u32,
-    ) -> (u64, u64) {
+    pub fn partial_threshold_jaccard_dist(&self, other: &PersistentCompactIntVec, threshold: u32) -> (u64, u64) {
         assert_eq!(self.n, other.len(), "length mismatch");
-        self.iter()
-            .zip(other.iter())
+        self.iter().zip(other.iter())
             .fold((0u64, 0u64), |(inter, uni), (a, b)| {
                 let ap = a >= threshold;
                 let bp = b >= threshold;
@@ -340,41 +215,12 @@ impl PersistentCompactIntVec {
     pub fn jaccard_dist(&self, other: &PersistentCompactIntVec) -> f64 {
         self.threshold_jaccard_dist(other, 1)
     }
-
-    pub fn iter(&self) -> Iter<'_> {
-        Iter {
-            pciv: self,
-            slot: 0,
-            overflow_pos: 0,
-        }
-    }
-}
-
-// ── IntSlice impl ─────────────────────────────────────────────────────────────
-
-use crate::traits::IntSlice;
-
-impl IntSlice for PersistentCompactIntVec {
-    fn len(&self) -> usize { self.n }
-    fn get(&self, slot: usize) -> u32 { self.get(slot) }
-    fn primary_bytes(&self) -> &[u8] {
-        &self.mmap[self.primary_offset..self.primary_offset + self.n]
-    }
-    fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_ {
-        (0..self.n_overflow).map(|i| (self.data_slot(i), self.data_value(i)))
-    }
-    fn iter(&self) -> impl Iterator<Item = u32> + '_ { self.iter() }
-    fn sum(&self) -> u64 { self.sum() }
-    fn count_nonzero(&self) -> u64 { self.count_nonzero() }
 }
 
 impl<'a> IntoIterator for &'a PersistentCompactIntVec {
     type Item = u32;
     type IntoIter = Iter<'a>;
-
-    fn into_iter(self) -> Iter<'a> {
-        self.iter()
-    }
+    fn into_iter(self) -> Iter<'a> { self.iter() }
 }
 
 pub struct Iter<'a> {
@@ -389,9 +235,7 @@ impl Iterator for Iter<'_> {
     type Item = u32;
 
     fn next(&mut self) -> Option<u32> {
-        if self.slot >= self.pciv.n {
-            return None;
-        }
+        if self.slot >= self.pciv.n { return None; }
         let v = self.pciv.mmap[self.pciv.primary_offset + self.slot];
         self.slot += 1;
         if v < 255 {

src/obicompactvec/src/tempbitvec.rs

@@ -4,66 +4,108 @@ use std::path::Path;
 use tempfile::TempDir;
 
 use crate::bitvec::{PersistentBitVec, PersistentBitVecBuilder};
-use crate::traits::{BitSlice, BitSliceMut};
+use crate::views::{BitSliceIter, BitSliceView, IntSliceView};
 
 // ── TempBitVec — frozen read-only, auto-deleted on drop ──────────────────────
 
-/// A bit vector backed by a temporary file.
-/// Implements [`BitSlice`]; the file is deleted when this value is dropped.
-/// Call [`make_persistent`](Self::make_persistent) to promote to a durable file.
 pub struct TempBitVec {
-    vec:   PersistentBitVec,
+    vec: PersistentBitVec,
     // Dropped after `vec` (field order), so the mmap is released before the
     // temp directory is deleted.
     _temp: TempDir,
 }
 
 impl TempBitVec {
-    /// Copy to a permanent file and open as a [`PersistentBitVec`].
     pub fn make_persistent(&self, path: &Path) -> io::Result<PersistentBitVec> {
         std::fs::copy(self.vec.path(), path)?;
         PersistentBitVec::open(path)
     }
 
-    pub fn len(&self)      -> usize { self.vec.len() }
-    pub fn is_empty(&self) -> bool  { self.vec.is_empty() }
-}
-
-impl BitSlice for TempBitVec {
-    fn len(&self)   -> usize  { self.vec.len() }
-    fn words(&self) -> &[u64] { self.vec.words() }
+    pub fn len(&self) -> usize {
+        self.vec.len()
+    }
+    pub fn is_empty(&self) -> bool {
+        self.vec.is_empty()
+    }
+    pub fn get(&self, slot: usize) -> bool {
+        self.vec.get(slot)
+    }
+    pub fn count_ones(&self) -> u64 {
+        self.vec.count_ones()
+    }
+    pub fn view(&self) -> BitSliceView<'_> {
+        self.vec.view()
+    }
+    pub fn iter(&self) -> BitSliceIter<'_> {
+        self.view().iter()
+    }
 }
 
 // ── TempBitVecBuilder — mutable, becomes TempBitVec on freeze ────────────────
 
-/// Writable builder for a [`TempBitVec`].  `pub(crate)` — callers receive
-/// only the frozen result via [`freeze`](Self::freeze).
-pub(crate) struct TempBitVecBuilder {
+pub struct TempBitVecBuilder {
     builder: PersistentBitVecBuilder,
-    temp:    TempDir,
+    temp: TempDir,
 }
 
 impl TempBitVecBuilder {
-    pub(crate) fn new(n: usize) -> io::Result<Self> {
+    pub fn new(n: usize) -> io::Result<Self> {
         let temp = TempDir::new()?;
         let path = temp.path().join("data.pbiv");
         let builder = PersistentBitVecBuilder::new(n, &path)?;
         Ok(Self { builder, temp })
     }
 
-    /// Finalize writes and return a frozen, read-only [`TempBitVec`].
-    pub(crate) fn freeze(self) -> io::Result<TempBitVec> {
+    pub fn new_ones(n: usize) -> io::Result<Self> {
+        let temp = TempDir::new()?;
+        let path = temp.path().join("data.pbiv");
+        let builder = PersistentBitVecBuilder::new_ones(n, &path)?;
+        Ok(Self { builder, temp })
+    }
+
+    pub fn freeze(self) -> io::Result<TempBitVec> {
         let Self { builder, temp } = self;
         let vec = builder.finish()?;
         Ok(TempBitVec { vec, _temp: temp })
     }
-}
 
-impl BitSlice for TempBitVecBuilder {
-    fn len(&self)   -> usize  { self.builder.len() }
-    fn words(&self) -> &[u64] { self.builder.words() }
-}
+    pub fn set(&mut self, slot: usize, value: bool) {
+        self.builder.set(slot, value);
+    }
 
-impl BitSliceMut for TempBitVecBuilder {
-    fn words_mut(&mut self) -> &mut [u64] { self.builder.words_mut() }
+    pub fn view(&self) -> BitSliceView<'_> {
+        self.builder.view()
+    }
+
+    pub fn or(&mut self, other: BitSliceView<'_>) {
+        self.builder.or(other);
+    }
+
+    pub fn and(&mut self, other: BitSliceView<'_>) {
+        self.builder.and(other);
+    }
+
+    pub fn xor(&mut self, other: BitSliceView<'_>) {
+        self.builder.xor(other);
+    }
+
+    pub fn not(&mut self) {
+        self.builder.not();
+    }
+
+    pub fn copy_from(&mut self, src: BitSliceView<'_>) {
+        self.builder.copy_from(src);
+    }
+
+    pub fn or_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
+        self.builder.or_where(col, pred);
+    }
+
+    pub fn and_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
+        self.builder.and_where(col, pred);
+    }
+
+    pub fn xor_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
+        self.builder.xor_where(col, pred);
+    }
 }

src/obicompactvec/src/tempintvec.rs

@@ -5,13 +5,10 @@ use tempfile::TempDir;
 
 use crate::builder::PersistentCompactIntVecBuilder;
 use crate::reader::PersistentCompactIntVec;
-use crate::traits::{IntSlice, IntSliceMut};
+use crate::views::{BitSliceView, IntSliceView};
 
 // ── TempCompactIntVec — frozen read-only, auto-deleted on drop ────────────────
 
-/// A compact int vector backed by a temporary file.
-/// Implements [`IntSlice`]; the file is deleted when this value is dropped.
-/// Call [`make_persistent`](Self::make_persistent) to promote to a durable file.
 pub struct TempCompactIntVec {
     vec:   PersistentCompactIntVec,
     // Dropped after `vec` (field order), so the mmap is released before the
@@ -20,7 +17,6 @@ pub struct TempCompactIntVec {
 }
 
 impl TempCompactIntVec {
-    /// Copy to a permanent file and open as a [`PersistentCompactIntVec`].
     pub fn make_persistent(&self, path: &Path) -> io::Result<PersistentCompactIntVec> {
         std::fs::copy(self.vec.path(), path)?;
         PersistentCompactIntVec::open(path)
@@ -28,55 +24,66 @@ impl TempCompactIntVec {
 
     pub fn len(&self)      -> usize { self.vec.len() }
     pub fn is_empty(&self) -> bool  { self.vec.is_empty() }
-}
-
-impl IntSlice for TempCompactIntVec {
-    fn len(&self)              -> usize { self.vec.len() }
-    fn get(&self, slot: usize) -> u32   { self.vec.get(slot) }
-    fn primary_bytes(&self)    -> &[u8] { self.vec.primary_bytes() }
-    fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_ {
-        self.vec.overflow_entries()
-    }
-    fn sum(&self)           -> u64 { self.vec.sum() }
-    fn count_nonzero(&self) -> u64 { self.vec.count_nonzero() }
+    pub fn get(&self, slot: usize) -> u32  { self.vec.get(slot) }
+    pub fn sum(&self)      -> u64   { self.vec.sum() }
+    pub fn view(&self)     -> IntSliceView<'_> { self.vec.view() }
+    pub fn iter(&self) -> crate::reader::Iter<'_> { self.vec.iter() }
 }
 
 // ── TempCompactIntVecBuilder — mutable, becomes TempCompactIntVec on freeze ──
 
-/// Writable builder for a [`TempCompactIntVec`].  `pub(crate)` — callers
-/// receive only the frozen result via [`freeze`](Self::freeze).
-pub(crate) struct TempCompactIntVecBuilder {
+pub struct TempCompactIntVecBuilder {
     builder: PersistentCompactIntVecBuilder,
     temp:    TempDir,
 }
 
 impl TempCompactIntVecBuilder {
-    pub(crate) fn new(n: usize) -> io::Result<Self> {
+    pub fn new(n: usize) -> io::Result<Self> {
         let temp = TempDir::new()?;
         let path = temp.path().join("data.pciv");
         let builder = PersistentCompactIntVecBuilder::new(n, &path)?;
         Ok(Self { builder, temp })
     }
 
-    /// Finalize writes and return a frozen, read-only [`TempCompactIntVec`].
-    pub(crate) fn freeze(self) -> io::Result<TempCompactIntVec> {
+    pub fn freeze(self) -> io::Result<TempCompactIntVec> {
         let Self { builder, temp } = self;
         let vec = builder.finish()?;
         Ok(TempCompactIntVec { vec, _temp: temp })
     }
-}
 
-impl IntSlice for TempCompactIntVecBuilder {
-    fn len(&self)              -> usize { self.builder.len() }
-    fn get(&self, slot: usize) -> u32   { self.builder.get(slot) }
-    fn primary_bytes(&self)    -> &[u8] { self.builder.primary_bytes() }
-    fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_ {
-        self.builder.overflow_entries()
+    pub fn n(&self) -> usize { self.builder.len() }
+
+    pub fn set(&mut self, slot: usize, value: u32) { self.builder.set(slot, value); }
+    pub fn get(&self, slot: usize) -> u32           { self.builder.get(slot) }
+
+    pub fn primary_bytes(&self)         -> &[u8]      { self.builder.primary_bytes() }
+    pub fn primary_bytes_mut(&mut self) -> &mut [u8]  { self.builder.primary_bytes_mut() }
+
+    pub fn inc_present(&mut self, col: BitSliceView<'_>) {
+        self.builder.inc_present(col);
     }
-}
 
-impl IntSliceMut for TempCompactIntVecBuilder {
-    fn set(&mut self, slot: usize, value: u32) { self.builder.set(slot, value); }
-    fn primary_bytes_mut(&mut self) -> &mut [u8] { self.builder.primary_bytes_mut() }
-    fn clear_overflow(&mut self)                  { self.builder.clear_overflow(); }
+    pub fn inc_present_fast(&mut self, col: BitSliceView<'_>) {
+        self.builder.inc_present_fast(col);
+    }
+
+    pub fn inc_predicate(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
+        self.builder.inc_predicate(col, pred);
+    }
+
+    pub fn inc_predicate_fast(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
+        self.builder.inc_predicate_fast(col, pred);
+    }
+
+    pub fn add(&mut self, other: IntSliceView<'_>) {
+        self.builder.add(other);
+    }
+
+    pub fn mask_with(&mut self, mask: BitSliceView<'_>) {
+        self.builder.mask_with(mask);
+    }
+
+    pub fn min(&mut self, other: IntSliceView<'_>)  { self.builder.min(other); }
+    pub fn max(&mut self, other: IntSliceView<'_>)  { self.builder.max(other); }
+    pub fn diff(&mut self, other: IntSliceView<'_>) { self.builder.diff(other); }
 }

src/obicompactvec/src/tests/bitmatrix.rs

@@ -1,7 +1,7 @@
 use tempfile::tempdir;
 
 use crate::{pack_bit_matrix, PersistentBitMatrix, PersistentBitMatrixBuilder};
-use crate::traits::{BitPartials, BitSlice, BitSliceMut};
+use crate::traits::BitPartials;
 
 fn make_matrix(cols: &[&[bool]]) -> (tempfile::TempDir, PersistentBitMatrix) {
     let n = cols.first().map_or(0, |c| c.len());

src/obicompactvec/src/tests/bitvec.rs

@@ -1,6 +1,5 @@
 use tempfile::tempdir;
 
-use crate::traits::{BitSlice, BitSliceMut};
 use crate::{PersistentBitVec, PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder};
 
 fn make_bv(bits: &[bool]) -> (tempfile::TempDir, PersistentBitVec) {
@@ -78,7 +77,7 @@ fn op_and() {
     let dir = tempdir().unwrap();
     let path = dir.path().join("out.pbiv");
     let mut b = PersistentBitVecBuilder::build_from(&ra, &path).unwrap();
-    b.and(&rb);
+    b.and(rb.view());
     b.close().unwrap();
     let r = PersistentBitVec::open(&path).unwrap();
     assert_eq!(r.iter().collect::<Vec<_>>(), vec![true, false, false, false]);
@@ -91,7 +90,7 @@ fn op_or() {
     let dir = tempdir().unwrap();
     let path = dir.path().join("out.pbiv");
     let mut b = PersistentBitVecBuilder::build_from(&ra, &path).unwrap();
-    b.or(&rb);
+    b.or(rb.view());
     b.close().unwrap();
     let r = PersistentBitVec::open(&path).unwrap();
     assert_eq!(r.iter().collect::<Vec<_>>(), vec![true, true, true, false]);
@@ -104,7 +103,7 @@ fn op_xor() {
     let dir = tempdir().unwrap();
     let path = dir.path().join("out.pbiv");
     let mut b = PersistentBitVecBuilder::build_from(&ra, &path).unwrap();
-    b.xor(&rb);
+    b.xor(rb.view());
     b.close().unwrap();
     let r = PersistentBitVec::open(&path).unwrap();
     assert_eq!(r.iter().collect::<Vec<_>>(), vec![false, true, true, false]);

src/obicompactvec/src/tests/colgroup.rs

@@ -5,8 +5,7 @@ use crate::{
     PersistentBitMatrix, PersistentBitMatrixBuilder,
     PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
 };
-use crate::traits::{BitSlice, BitSliceMut, IntSlice, IntSliceMut};
-use crate::{MemoryBitVec, MemoryIntVec};
+use crate::{PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder};
 
 // ── helpers ───────────────────────────────────────────────────────────────────
 
@@ -114,42 +113,51 @@ fn int_partial_group_any() {
 #[test]
 fn mask_with_zeros_selected_slots() {
     // count vec [10, 20, 30, 40], mask [T, F, T, F] → [10, 0, 30, 0]
-    let mut v = MemoryIntVec::new(4);
+    let dir = tempdir().unwrap();
+    let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
     v.set(0, 10); v.set(1, 20); v.set(2, 30); v.set(3, 40);
-    let mut mask = MemoryBitVec::new(4);
+    let mut mask = PersistentBitVecBuilder::new(4, &dir.path().join("m.pbiv")).unwrap();
     mask.set(0, true); mask.set(2, true);
-    v.mask_with(&mask);
-    assert_eq!(v.get(0), 10);
-    assert_eq!(v.get(1), 0);
-    assert_eq!(v.get(2), 30);
-    assert_eq!(v.get(3), 0);
+    v.mask_with(mask.view());
+    v.close().unwrap();
+    let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
+    assert_eq!(r.get(0), 10);
+    assert_eq!(r.get(1), 0);
+    assert_eq!(r.get(2), 30);
+    assert_eq!(r.get(3), 0);
 }
 
 #[test]
 fn mask_with_overflow_slot_zeroed() {
     // overflow slot (value 500) masked out → removed from overflow, primary=0
-    let mut v = MemoryIntVec::new(3);
+    let dir = tempdir().unwrap();
+    let mut v = PersistentCompactIntVecBuilder::new(3, &dir.path().join("v.pciv")).unwrap();
     v.set(0, 10); v.set(1, 500); v.set(2, 5);
-    let mut mask = MemoryBitVec::new(3);
+    let mut mask = PersistentBitVecBuilder::new(3, &dir.path().join("m.pbiv")).unwrap();
     mask.set(0, true); mask.set(2, true);  // slot 1 masked out
-    v.mask_with(&mask);
-    assert_eq!(v.get(0), 10);
-    assert_eq!(v.get(1), 0);
-    assert_eq!(v.get(2), 5);
-    let ov: Vec<_> = v.overflow_entries().collect();
+    v.mask_with(mask.view());
+    v.close().unwrap();
+    let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
+    assert_eq!(r.get(0), 10);
+    assert_eq!(r.get(1), 0);
+    assert_eq!(r.get(2), 5);
+    let ov: Vec<_> = r.view().overflow_entries().collect();
     assert!(ov.is_empty(), "overflow entry for masked-out slot should be gone");
 }
 
 #[test]
 fn mask_with_all_ones_is_noop() {
-    let mut v = MemoryIntVec::new(4);
+    let dir = tempdir().unwrap();
+    let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
     v.set(0, 300); v.set(1, 1); v.set(2, 0); v.set(3, 42);
-    let mask = MemoryBitVec::ones(4);
-    v.mask_with(&mask);
-    assert_eq!(v.get(0), 300);
-    assert_eq!(v.get(1), 1);
-    assert_eq!(v.get(2), 0);
-    assert_eq!(v.get(3), 42);
+    let mask = PersistentBitVecBuilder::new_ones(4, &dir.path().join("m.pbiv")).unwrap();
+    v.mask_with(mask.view());
+    v.close().unwrap();
+    let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
+    assert_eq!(r.get(0), 300);
+    assert_eq!(r.get(1), 1);
+    assert_eq!(r.get(2), 0);
+    assert_eq!(r.get(3), 42);
 }
 
 // ── BitMatrix: partial_group_presence_count ───────────────────────────────────

src/obicompactvec/src/tests/intmatrix.rs

@@ -1,7 +1,7 @@
 use tempfile::tempdir;
 
 use crate::{pack_compact_int_matrix, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder};
-use crate::traits::{CountPartials, IntSlice};
+use crate::traits::CountPartials;
 
 fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
     let n = cols.first().map_or(0, |c| c.len());
@@ -290,7 +290,7 @@ fn col_view_packed_matches_columnar() {
         }
         assert_eq!(col_view.sum(), col_ref.sum(), "col={c} sum");
         let mut ov_view: Vec<(usize, u32)> = col_view.overflow_entries().collect();
-        let mut ov_ref:  Vec<(usize, u32)> = col_ref.overflow_entries().collect();
+        let mut ov_ref:  Vec<(usize, u32)> = col_ref.view().overflow_entries().collect();
         ov_view.sort_unstable_by_key(|&(s, _)| s);
         ov_ref.sort_unstable_by_key(|&(s, _)| s);
         assert_eq!(ov_view, ov_ref, "col={c} overflow_entries");

src/obicompactvec/src/tests/memoryvec.rs

@@ -1,484 +0,0 @@
-use tempfile::tempdir;
-
-use crate::traits::{BitSlice, BitSliceMut, BitToInt, IntSlice, IntSliceMut, IntToBit};
-use crate::{MemoryBitVec, MemoryIntVec, PersistentBitVec, PersistentBitVecBuilder};
-
-// ── MemoryBitVec ──────────────────────────────────────────────────────────────
-
-#[test]
-fn mbv_new_all_zero() {
-    let v = MemoryBitVec::new(10);
-    assert_eq!(v.len(), 10);
-    assert!(!(0..10).any(|s| v.get(s)));
-    assert_eq!(v.count_ones(), 0);
-}
-
-#[test]
-fn mbv_ones_all_set() {
-    let v = MemoryBitVec::ones(10);
-    assert!((0..10).all(|s| v.get(s)));
-    assert_eq!(v.count_ones(), 10);
-    assert_eq!(v.count_zeros(), 0);
-}
-
-#[test]
-fn mbv_ones_no_padding_leak() {
-    // 5 bits: padding bits in last word must stay 0
-    let v = MemoryBitVec::ones(5);
-    assert_eq!(v.words()[0], 0b11111);
-}
-
-#[test]
-fn mbv_set_get_roundtrip() {
-    let mut v = MemoryBitVec::new(64);
-    v.set(0, true);
-    v.set(63, true);
-    assert!(v.get(0));
-    assert!(!v.get(1));
-    assert!(v.get(63));
-    assert_eq!(v.count_ones(), 2);
-}
-
-#[test]
-fn mbv_and() {
-    let mut a = MemoryBitVec::new(4);
-    a.set(0, true); a.set(1, true);
-    let mut b = MemoryBitVec::new(4);
-    b.set(0, true); b.set(2, true);
-    a.and(&b);
-    assert!(a.get(0)); assert!(!a.get(1)); assert!(!a.get(2));
-}
-
-#[test]
-fn mbv_or() {
-    let mut a = MemoryBitVec::new(4);
-    a.set(0, true); a.set(1, true);
-    let mut b = MemoryBitVec::new(4);
-    b.set(0, true); b.set(2, true);
-    a.or(&b);
-    assert!(a.get(0)); assert!(a.get(1)); assert!(a.get(2)); assert!(!a.get(3));
-}
-
-#[test]
-fn mbv_xor() {
-    let mut a = MemoryBitVec::new(4);
-    a.set(0, true); a.set(1, true);
-    let mut b = MemoryBitVec::new(4);
-    b.set(0, true); b.set(2, true);
-    a.xor(&b);
-    assert!(!a.get(0)); assert!(a.get(1)); assert!(a.get(2)); assert!(!a.get(3));
-}
-
-#[test]
-fn mbv_not() {
-    let mut a = MemoryBitVec::new(4);
-    a.set(0, true); a.set(2, true);
-    a.not();
-    assert!(!a.get(0)); assert!(a.get(1)); assert!(!a.get(2)); assert!(a.get(3));
-}
-
-#[test]
-fn mbv_not_no_padding_leak() {
-    let mut v = MemoryBitVec::new(5);
-    v.not();
-    assert_eq!(v.count_ones(), 5);
-    assert_eq!(v.words()[0], 0b11111);
-}
-
-#[test]
-fn mbv_ops_chaining() {
-    let mut a = MemoryBitVec::ones(8);
-    let b = MemoryBitVec::new(8); // all zeros
-    a.and(&b).or(&b).not();
-    assert_eq!(a.count_ones(), 8);
-}
-
-#[test]
-fn mbv_std_ops_owned() {
-    let mut a = MemoryBitVec::new(4);
-    a.set(0, true); a.set(1, true);
-    let mut b = MemoryBitVec::new(4);
-    b.set(1, true); b.set(2, true);
-    let c = a & &b;
-    assert!(!c.get(0)); assert!(c.get(1)); assert!(!c.get(2));
-}
-
-#[test]
-fn mbv_std_ops_assign() {
-    let mut a = MemoryBitVec::new(4);
-    a.set(0, true); a.set(1, true);
-    let mut b = MemoryBitVec::new(4);
-    b.set(1, true); b.set(2, true);
-    a &= &b;
-    assert!(!a.get(0)); assert!(a.get(1));
-}
-
-#[test]
-fn mbv_from_persistent() {
-    let dir = tempdir().unwrap();
-    let path = dir.path().join("v.pbiv");
-    let mut builder = PersistentBitVecBuilder::new(4, &path).unwrap();
-    builder.set(1, true); builder.set(3, true);
-    builder.close().unwrap();
-    let pv = PersistentBitVec::open(&path).unwrap();
-    let mv = MemoryBitVec::from(&pv);
-    assert!(!mv.get(0)); assert!(mv.get(1)); assert!(!mv.get(2)); assert!(mv.get(3));
-}
-
-#[test]
-fn mbv_persist_roundtrip() {
-    let dir = tempdir().unwrap();
-    let path = dir.path().join("out.pbiv");
-    let mut v = MemoryBitVec::new(8);
-    v.set(2, true); v.set(5, true);
-    let builder = v.persist(&path).unwrap();
-    builder.close().unwrap();
-    let pv = PersistentBitVec::open(&path).unwrap();
-    assert!(pv.get(2)); assert!(pv.get(5));
-    assert_eq!(pv.count_ones(), 2);
-}
-
-// ── MemoryIntVec ──────────────────────────────────────────────────────────────
-
-#[test]
-fn miv_new_all_zero() {
-    let v = MemoryIntVec::new(10);
-    assert_eq!(v.len(), 10);
-    assert!((0..10).all(|s| v.get(s) == 0));
-}
-
-#[test]
-fn miv_set_get_roundtrip() {
-    let mut v = MemoryIntVec::new(4);
-    v.set(0, 42); v.set(3, 200);
-    assert_eq!(v.get(0), 42);
-    assert_eq!(v.get(1), 0);
-    assert_eq!(v.get(3), 200);
-}
-
-#[test]
-fn miv_overflow_roundtrip() {
-    let mut v = MemoryIntVec::new(4);
-    v.set(1, 1000);
-    assert_eq!(v.get(1), 1000);
-    assert_eq!(v.get(0), 0);
-}
-
-#[test]
-fn miv_inc_dec() {
-    let mut v = MemoryIntVec::new(4);
-    v.inc(2); v.inc(2); v.inc(2);
-    assert_eq!(v.get(2), 3);
-    v.dec(2);
-    assert_eq!(v.get(2), 2);
-}
-
-#[test]
-fn miv_dec_saturates_at_zero() {
-    let mut v = MemoryIntVec::new(4);
-    v.dec(0);
-    assert_eq!(v.get(0), 0);
-}
-
-#[test]
-fn miv_add_at() {
-    let mut v = MemoryIntVec::new(4);
-    v.add_at(1, 100); v.add_at(1, 200);
-    assert_eq!(v.get(1), 300);
-}
-
-#[test]
-fn miv_min_max() {
-    let mut a = MemoryIntVec::new(4);
-    a.set(0, 5); a.set(1, 2); a.set(2, 8);
-    let mut b = MemoryIntVec::new(4);
-    b.set(0, 3); b.set(1, 7); b.set(2, 8);
-    let mut c = MemoryIntVec::from(&a);
-    IntSliceMut::min(&mut c, &b);
-    assert_eq!(c.get(0), 3); assert_eq!(c.get(1), 2); assert_eq!(c.get(2), 8);
-    let mut d = MemoryIntVec::from(&a);
-    IntSliceMut::max(&mut d, &b);
-    assert_eq!(d.get(0), 5); assert_eq!(d.get(1), 7); assert_eq!(d.get(2), 8);
-}
-
-#[test]
-fn miv_add_diff() {
-    let mut a = MemoryIntVec::new(3);
-    a.set(0, 10); a.set(1, 5);
-    let mut b = MemoryIntVec::new(3);
-    b.set(0, 3); b.set(1, 8);
-    let mut c = MemoryIntVec::from(&a);
-    c.add(&b);
-    assert_eq!(c.get(0), 13); assert_eq!(c.get(1), 13);
-    let mut d = MemoryIntVec::from(&a);
-    d.diff(&b);
-    assert_eq!(d.get(0), 7); assert_eq!(d.get(1), 0); // saturating sub
-}
-
-#[test]
-fn miv_std_ops() {
-    let mut a = MemoryIntVec::new(3);
-    a.set(0, 10); a.set(1, 5);
-    let mut b = MemoryIntVec::new(3);
-    b.set(0, 3); b.set(1, 8);
-    let c = &a + &b;
-    assert_eq!(c.get(0), 13); assert_eq!(c.get(1), 13);
-    let d = &a - &b;
-    assert_eq!(d.get(0), 7); assert_eq!(d.get(1), 0);
-}
-
-#[test]
-fn miv_from_persistent() {
-    use crate::{PersistentCompactIntVec, PersistentCompactIntVecBuilder};
-    let dir = tempdir().unwrap();
-    let path = dir.path().join("v.pciv");
-    let mut b = PersistentCompactIntVecBuilder::new(4, &path).unwrap();
-    b.set(1, 42); b.set(3, 1000);
-    b.close().unwrap();
-    let pv = PersistentCompactIntVec::open(&path).unwrap();
-    let mv = MemoryIntVec::from(&pv);
-    assert_eq!(mv.get(0), 0); assert_eq!(mv.get(1), 42); assert_eq!(mv.get(3), 1000);
-}
-
-// ── Cross-type conversions ────────────────────────────────────────────────────
-
-#[test]
-fn to_bitvec_threshold() {
-    let mut v = MemoryIntVec::new(5);
-    v.set(0, 0); v.set(1, 1); v.set(2, 5); v.set(3, 10); v.set(4, 3);
-    let bv = v.to_bitvec(4); // > 4: slots 2 (5) and 3 (10) pass
-    assert!(!bv.get(0)); assert!(!bv.get(1)); assert!(bv.get(2));
-    assert!(bv.get(3)); assert!(!bv.get(4));
-}
-
-#[test]
-fn to_presence() {
-    let mut v = MemoryIntVec::new(4);
-    v.set(1, 1); v.set(3, 100);
-    let bv = v.to_presence();
-    assert!(!bv.get(0)); assert!(bv.get(1)); assert!(!bv.get(2)); assert!(bv.get(3));
-}
-
-#[test]
-fn to_intvec_roundtrip() {
-    let mut bv = MemoryBitVec::new(8);
-    bv.set(0, true); bv.set(3, true); bv.set(7, true);
-    let iv = bv.to_intvec();
-    assert_eq!(iv.get(0), 1); assert_eq!(iv.get(1), 0);
-    assert_eq!(iv.get(3), 1); assert_eq!(iv.get(7), 1);
-}
-
-#[test]
-fn to_intvec_word_boundary() {
-    // 65 bits: spans two words
-    let mut bv = MemoryBitVec::new(65);
-    bv.set(63, true); bv.set(64, true);
-    let iv = bv.to_intvec();
-    assert_eq!(iv.get(63), 1); assert_eq!(iv.get(64), 1); assert_eq!(iv.get(62), 0);
-}
-
-#[test]
-fn count_bits_accumulates() {
-    let mut count = MemoryIntVec::new(8);
-    let mut b1 = MemoryBitVec::new(8);
-    b1.set(0, true); b1.set(2, true);
-    let mut b2 = MemoryBitVec::new(8);
-    b2.set(0, true); b2.set(3, true);
-    let mut b3 = MemoryBitVec::new(8);
-    b3.set(2, true); b3.set(3, true);
-    count.count_bits(&b1).count_bits(&b2).count_bits(&b3);
-    assert_eq!(count.get(0), 2);
-    assert_eq!(count.get(2), 2);
-    assert_eq!(count.get(3), 2);
-    assert_eq!(count.get(1), 0);
-}
-
-#[test]
-fn count_bits_skips_zero_words() {
-    // Entire first word is zero — should not touch those slots
-    let mut count = MemoryIntVec::new(128);
-    let mut bv = MemoryBitVec::new(128);
-    bv.set(64, true); bv.set(127, true);
-    count.count_bits(&bv);
-    assert_eq!(count.get(0), 0);
-    assert_eq!(count.get(64), 1);
-    assert_eq!(count.get(127), 1);
-}
-
-// ── min / max / add / diff — overflow edge cases ──────────────────────────────
-
-#[test]
-fn miv_min_overflow_edges() {
-    // [300, 50, 400, 300] min [50, 300, 500, 200]
-    // slot 0: self=overflow(300), other=primary(50)  → 50   (overflow removed)
-    // slot 1: self=primary(50),   other=overflow(300) → 50   (no overflow created)
-    // slot 2: self=overflow(400), other=overflow(500) → 400  (overflow updated)
-    // slot 3: self=overflow(300), other=primary(200)  → 200  (overflow removed, 200 < 255)
-    let mut a = MemoryIntVec::new(4);
-    a.set(0, 300); a.set(1, 50); a.set(2, 400); a.set(3, 300);
-    let mut b = MemoryIntVec::new(4);
-    b.set(0, 50); b.set(1, 300); b.set(2, 500); b.set(3, 200);
-    IntSliceMut::min(&mut a, &b);
-    assert_eq!(a.get(0), 50);
-    assert_eq!(a.get(1), 50);
-    assert_eq!(a.get(2), 400);
-    assert_eq!(a.get(3), 200);
-    // Only slot 2 should still have an overflow entry.
-    let ov: std::collections::HashMap<usize, u32> = a.overflow_entries().collect();
-    assert_eq!(ov.len(), 1);
-    assert_eq!(ov[&2], 400);
-}
-
-#[test]
-fn miv_max_overflow_edges() {
-    // [50, 300, 100, 400] max [300, 50, 500, 200]
-    // slot 0: self=primary(50),   other=overflow(300) → 300  (overflow created)
-    // slot 1: self=overflow(300), other=primary(50)   → 300  (overflow unchanged)
-    // slot 2: self=primary(100),  other=overflow(500) → 500  (overflow created)
-    // slot 3: self=overflow(400), other=overflow(200) → 400  (overflow unchanged, 200 < 255 wait...)
-    // Wait — 200 < 255 so other slot 3 is NOT overflow. Correct: max(400, 200) = 400.
-    let mut a = MemoryIntVec::new(4);
-    a.set(0, 50); a.set(1, 300); a.set(2, 100); a.set(3, 400);
-    let mut b = MemoryIntVec::new(4);
-    b.set(0, 300); b.set(1, 50); b.set(2, 500); b.set(3, 200);
-    IntSliceMut::max(&mut a, &b);
-    assert_eq!(a.get(0), 300);
-    assert_eq!(a.get(1), 300);
-    assert_eq!(a.get(2), 500);
-    assert_eq!(a.get(3), 400);
-    let ov: std::collections::HashMap<usize, u32> = a.overflow_entries().collect();
-    assert_eq!(ov.len(), 4); // all four results >= 255
-    assert_eq!(ov[&0], 300);
-    assert_eq!(ov[&1], 300);
-    assert_eq!(ov[&2], 500);
-    assert_eq!(ov[&3], 400);
-}
-
-#[test]
-fn miv_add_overflow_edges() {
-    // [300, 50, 400, 200] + [50, 300, 200, 200]
-    // slot 0: self=overflow(300), other=primary(50)   → 350  (overflow updated)
-    // slot 1: self=primary(50),   other=overflow(300) → 350  (overflow created from primary)
-    // slot 2: self=overflow(400), other=overflow(200... wait 200 < 255)
-    //         other slot 2 is primary(200); 400+200=600 (overflow updated)
-    // slot 3: self=primary(200),  other=primary(200)  → 400  (overflow created, 400 >= 255)
-    let mut a = MemoryIntVec::new(4);
-    a.set(0, 300); a.set(1, 50); a.set(2, 400); a.set(3, 200);
-    let mut b = MemoryIntVec::new(4);
-    b.set(0, 50); b.set(1, 300); b.set(2, 200); b.set(3, 200);
-    a.add(&b);
-    assert_eq!(a.get(0), 350);
-    assert_eq!(a.get(1), 350);
-    assert_eq!(a.get(2), 600);
-    assert_eq!(a.get(3), 400);
-    let ov: std::collections::HashMap<usize, u32> = a.overflow_entries().collect();
-    assert_eq!(ov.len(), 4);
-}
-
-#[test]
-fn miv_add_both_overflow() {
-    // [300] + [400] = [700]
-    let mut a = MemoryIntVec::new(1);
-    a.set(0, 300);
-    let mut b = MemoryIntVec::new(1);
-    b.set(0, 400);
-    a.add(&b);
-    assert_eq!(a.get(0), 700);
-    let ov: std::collections::HashMap<usize, u32> = a.overflow_entries().collect();
-    assert_eq!(ov[&0], 700);
-}
-
-#[test]
-fn miv_diff_overflow_edges() {
-    // [300, 400, 400, 50] - [100, 50, 350, 300]
-    // slot 0: self=overflow(300), other=primary(100)   → 200  (overflow removed, 200 < 255)
-    // slot 1: self=overflow(400), other=primary(50)    → 350  (overflow updated, 350 >= 255)
-    // slot 2: self=overflow(400), other=overflow(350)  → 50   (overflow removed, 50 < 255)
-    // slot 3: self=primary(50),   other=overflow(300)  → 0    (saturating, stays primary)
-    let mut a = MemoryIntVec::new(4);
-    a.set(0, 300); a.set(1, 400); a.set(2, 400); a.set(3, 50);
-    let mut b = MemoryIntVec::new(4);
-    b.set(0, 100); b.set(1, 50); b.set(2, 350); b.set(3, 300);
-    a.diff(&b);
-    assert_eq!(a.get(0), 200);
-    assert_eq!(a.get(1), 350);
-    assert_eq!(a.get(2), 50);
-    assert_eq!(a.get(3), 0);
-    let ov: std::collections::HashMap<usize, u32> = a.overflow_entries().collect();
-    assert_eq!(ov.len(), 1); // only slot 1 remains overflow
-    assert_eq!(ov[&1], 350);
-}
-
-// ── Comparison operators ──────────────────────────────────────────────────────
-
-#[test]
-fn cmp_gt() {
-    let mut v = MemoryIntVec::new(5);
-    v.set(0, 0); v.set(1, 3); v.set(2, 5); v.set(3, 3); v.set(4, 10);
-    let bv = v.gt(3);
-    assert!(!bv.get(0)); assert!(!bv.get(1)); assert!(bv.get(2));
-    assert!(!bv.get(3)); assert!(bv.get(4));
-}
-
-#[test]
-fn cmp_geq() {
-    let mut v = MemoryIntVec::new(4);
-    v.set(0, 2); v.set(1, 3); v.set(2, 4); v.set(3, 1);
-    let bv = v.geq(3);
-    assert!(!bv.get(0)); assert!(bv.get(1)); assert!(bv.get(2)); assert!(!bv.get(3));
-}
-
-#[test]
-fn cmp_lt() {
-    let mut v = MemoryIntVec::new(4);
-    v.set(0, 2); v.set(1, 3); v.set(2, 4); v.set(3, 0);
-    let bv = v.lt(3);
-    assert!(bv.get(0)); assert!(!bv.get(1)); assert!(!bv.get(2)); assert!(bv.get(3));
-}
-
-#[test]
-fn cmp_leq() {
-    let mut v = MemoryIntVec::new(4);
-    v.set(0, 2); v.set(1, 3); v.set(2, 4); v.set(3, 3);
-    let bv = v.leq(3);
-    assert!(bv.get(0)); assert!(bv.get(1)); assert!(!bv.get(2)); assert!(bv.get(3));
-}
-
-#[test]
-fn cmp_scalar_with_overflow() {
-    // Slots: [10, 1000, 50, 500, 0]
-    // geq(100): slots 1 (1000) and 3 (500) → both overflow, must qualify
-    // lt(500):  slots 0 (10), 2 (50), 4 (0) → primary; slot 1 (1000) → no; slot 3 (500) → no
-    // geq(2000): only slot 1 (1000) fails, no slot qualifies
-    let mut v = MemoryIntVec::new(5);
-    v.set(0, 10); v.set(1, 1000); v.set(2, 50); v.set(3, 500); v.set(4, 0);
-
-    let bv = v.geq(100);
-    assert!(!bv.get(0)); assert!(bv.get(1)); assert!(!bv.get(2));
-    assert!(bv.get(3)); assert!(!bv.get(4));
-
-    let bv = v.lt(500);
-    assert!(bv.get(0)); assert!(!bv.get(1)); assert!(bv.get(2));
-    assert!(!bv.get(3)); assert!(bv.get(4));
-
-    let bv = v.geq(2000);
-    assert!(!(0..5).any(|s| bv.get(s)));
-}
-
-#[test]
-fn filter_pattern() {
-    // Typical filter: ingroup >= min_count AND outgroup <= max_outgroup
-    let mut ingroup  = MemoryIntVec::new(6);
-    let mut outgroup = MemoryIntVec::new(6);
-    // slot 2: ingroup=3, outgroup=0  → keep
-    // slot 4: ingroup=2, outgroup=1  → drop (outgroup > 0)
-    // slot 5: ingroup=1, outgroup=0  → drop (ingroup < 2)
-    ingroup.set(2, 3); ingroup.set(4, 2); ingroup.set(5, 1);
-    outgroup.set(4, 1);
-    let out_mask  = outgroup.leq(0);
-    let mut in_mask = ingroup.geq(2);
-    let keep = in_mask.and(&out_mask);
-    assert!(!keep.get(0)); assert!(!keep.get(1));
-    assert!(keep.get(2));
-    assert!(!keep.get(4)); assert!(!keep.get(5));
-}

src/obicompactvec/src/tests/mod.rs

@@ -2,12 +2,9 @@ mod bitmatrix;
 mod bitvec;
 mod colgroup;
 mod intmatrix;
-mod memoryvec;
 
 use tempfile::tempdir;
 
-use crate::traits::IntSliceMut;
-
 use crate::{PersistentCompactIntVec, PersistentCompactIntVecBuilder};
 
 fn roundtrip(values: &[(usize, u32)], n: usize) -> Vec<u32> {
@@ -173,7 +170,7 @@ fn combine_min() {
     let dir = tempdir().unwrap();
     let path = dir.path().join("out.pciv");
     let mut b = PersistentCompactIntVecBuilder::build_from(&ra, &path).unwrap();
-    b.min(&rb);
+    b.min(rb.view());
     b.close().unwrap();
     let r = PersistentCompactIntVec::open(&path).unwrap();
     assert_eq!(r.iter().collect::<Vec<_>>(), vec![10, 100, 0, 800]);
@@ -186,7 +183,7 @@ fn combine_max() {
     let dir = tempdir().unwrap();
     let path = dir.path().join("out.pciv");
     let mut b = PersistentCompactIntVecBuilder::build_from(&ra, &path).unwrap();
-    b.max(&rb);
+    b.max(rb.view());
     b.close().unwrap();
     let r = PersistentCompactIntVec::open(&path).unwrap();
     assert_eq!(r.iter().collect::<Vec<_>>(), vec![20, 300, 500, 1000]);
@@ -199,7 +196,7 @@ fn combine_add() {
     let dir = tempdir().unwrap();
     let path = dir.path().join("out.pciv");
     let mut b = PersistentCompactIntVecBuilder::build_from(&ra, &path).unwrap();
-    b.add(&rb);
+    b.add(rb.view());
     b.close().unwrap();
     let r = PersistentCompactIntVec::open(&path).unwrap();
     assert_eq!(r.iter().collect::<Vec<_>>(), vec![30, 300, 5, 101]);
@@ -224,7 +221,7 @@ fn combine_diff() {
     let dir = tempdir().unwrap();
     let path = dir.path().join("out.pciv");
     let mut b = PersistentCompactIntVecBuilder::build_from(&ra, &path).unwrap();
-    b.diff(&rb);
+    b.diff(rb.view());
     b.close().unwrap();
     let r = PersistentCompactIntVec::open(&path).unwrap();
     assert_eq!(r.iter().collect::<Vec<_>>(), vec![10, 700, 0, 0]);

src/obicompactvec/src/traits.rs

@@ -1,353 +1,5 @@
-use std::collections::HashMap;
-
 use ndarray::{Array1, Array2};
 
-// ── BitSlice / BitSliceMut ────────────────────────────────────────────────────
-
-/// Read-only view over the u64 word array of a bit vector.
-///
-/// Bit `i` is in `words()[i >> 6]` at position `i & 63`.
-/// Padding bits in the last word are zero.
-pub trait BitSlice {
-    fn len(&self) -> usize;
-    fn words(&self) -> &[u64];
-    fn is_empty(&self) -> bool { self.len() == 0 }
-    fn get(&self, slot: usize) -> bool {
-        (self.words()[slot >> 6] >> (slot & 63)) & 1 != 0
-    }
-    fn count_ones(&self) -> u64 {
-        self.words().iter().map(|w| w.count_ones() as u64).sum()
-    }
-    fn count_zeros(&self) -> u64 { self.len() as u64 - self.count_ones() }
-    fn partial_jaccard_dist<S: BitSlice>(&self, other: &S) -> (u64, u64) {
-        assert_eq!(self.len(), other.len(), "length mismatch");
-        self.words().iter().zip(other.words())
-            .fold((0u64, 0u64), |(i, u), (&a, &b)| {
-                (i + (a & b).count_ones() as u64, u + (a | b).count_ones() as u64)
-            })
-    }
-    fn jaccard_dist<S: BitSlice>(&self, other: &S) -> f64 {
-        let (inter, union) = self.partial_jaccard_dist(other);
-        if union == 0 { 0.0 } else { 1.0 - inter as f64 / union as f64 }
-    }
-    fn hamming_dist<S: BitSlice>(&self, other: &S) -> u64 {
-        assert_eq!(self.len(), other.len(), "length mismatch");
-        self.words().iter().zip(other.words())
-            .map(|(&a, &b)| (a ^ b).count_ones() as u64)
-            .sum()
-    }
-}
-
-/// Mutable view over a bit-vector word array; default methods maintain the
-/// zero-padding invariant on the last word.
-pub trait BitSliceMut: BitSlice {
-    fn words_mut(&mut self) -> &mut [u64];
-
-    fn set(&mut self, slot: usize, value: bool) {
-        let bit = 1u64 << (slot & 63);
-        if value { self.words_mut()[slot >> 6] |= bit; } else { self.words_mut()[slot >> 6] &= !bit; }
-    }
-
-    fn copy_from<S: BitSlice>(&mut self, src: &S) -> &mut Self {
-        assert_eq!(self.len(), src.len(), "BitSlice length mismatch");
-        self.words_mut().copy_from_slice(src.words());
-        self
-    }
-
-    fn and<S: BitSlice>(&mut self, other: &S) -> &mut Self {
-        assert_eq!(self.len(), other.len(), "BitSlice length mismatch");
-        for (w, &o) in self.words_mut().iter_mut().zip(other.words()) { *w &= o; }
-        self
-    }
-
-    fn or<S: BitSlice>(&mut self, other: &S) -> &mut Self {
-        assert_eq!(self.len(), other.len(), "BitSlice length mismatch");
-        for (w, &o) in self.words_mut().iter_mut().zip(other.words()) { *w |= o; }
-        self
-    }
-
-    fn xor<S: BitSlice>(&mut self, other: &S) -> &mut Self {
-        assert_eq!(self.len(), other.len(), "BitSlice length mismatch");
-        for (w, &o) in self.words_mut().iter_mut().zip(other.words()) { *w ^= o; }
-        self
-    }
-
-    fn not(&mut self) -> &mut Self {
-        let rem = self.len() % 64;
-        let words = self.words_mut();
-        for w in words.iter_mut() { *w ^= u64::MAX; }
-        if rem != 0 {
-            if let Some(last) = words.last_mut() { *last &= (1u64 << rem) - 1; }
-        }
-        self
-    }
-}
-
-// ── IntSlice / IntSliceMut ────────────────────────────────────────────────────
-
-/// Read-only access to a compact integer vector (values encoded as u32).
-pub trait IntSlice {
-    fn len(&self) -> usize;
-    fn get(&self, slot: usize) -> u32;
-    /// Raw primary byte slice (sentinel 255 marks overflow slots).
-    fn primary_bytes(&self) -> &[u8];
-    /// Iterator over `(slot, true_value)` pairs for all overflow entries (value >= 255).
-    fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_;
-    fn is_empty(&self) -> bool { self.len() == 0 }
-    fn iter(&self) -> impl Iterator<Item = u32> + '_ { (0..self.len()).map(|i| self.get(i)) }
-    fn sum(&self) -> u64 { self.iter().map(|v| v as u64).sum() }
-    fn count_nonzero(&self) -> u64 { self.iter().filter(|v| *v > 0).count() as u64 }
-
-    fn lt(&self, threshold: u32) -> MemoryBitVec { self.cmp_scalar(|v| v <  threshold) }
-    fn leq(&self, threshold: u32) -> MemoryBitVec { self.cmp_scalar(|v| v <= threshold) }
-    fn gt(&self, threshold: u32) -> MemoryBitVec  { self.cmp_scalar(|v| v >  threshold) }
-    fn geq(&self, threshold: u32) -> MemoryBitVec { self.cmp_scalar(|v| v >= threshold) }
-
-    fn cmp_scalar(&self, pred: impl Fn(u32) -> bool) -> MemoryBitVec {
-        let n = self.len();
-        let mut words = vec![0u64; n.div_ceil(64)];
-        let primary = self.primary_bytes();
-        // Pass 1: byte scan — no HashMap access, vectorisable for simple predicates.
-        // Overflow slots (b == 255) are left as 0 and fixed in pass 2.
-        for s in 0..n {
-            let b = primary[s];
-            if b < 255 && pred(b as u32) {
-                words[s >> 6] |= 1u64 << (s & 63);
-            }
-        }
-        // Pass 2: fix up overflow slots — O(k), negligible.
-        for (s, val) in self.overflow_entries() {
-            if pred(val) { words[s >> 6] |= 1u64 << (s & 63); }
-        }
-        MemoryBitVec::from_words(words, n)
-    }
-}
-
-/// Mutable access; default methods use only `get` / `set` and maintain the
-/// compact encoding invariants on the implementor's side.
-pub trait IntSliceMut: IntSlice {
-    fn set(&mut self, slot: usize, value: u32);
-    fn primary_bytes_mut(&mut self) -> &mut [u8];
-    fn clear_overflow(&mut self);
-
-    fn inc(&mut self, slot: usize) -> &mut Self {
-        let v = self.get(slot);
-        self.set(slot, v.saturating_add(1));
-        self
-    }
-
-    fn dec(&mut self, slot: usize) -> &mut Self {
-        let v = self.get(slot);
-        self.set(slot, v.saturating_sub(1));
-        self
-    }
-
-    fn add_at(&mut self, slot: usize, delta: u32) -> &mut Self {
-        let v = self.get(slot);
-        self.set(slot, v.saturating_add(delta));
-        self
-    }
-
-    fn copy_from<S: IntSlice>(&mut self, src: &S) -> &mut Self {
-        assert_eq!(self.len(), src.len(), "IntSlice length mismatch");
-        self.primary_bytes_mut().copy_from_slice(src.primary_bytes());
-        self.clear_overflow();
-        for (slot, val) in src.overflow_entries() { self.set(slot, val); }
-        self
-    }
-
-    fn min<S: IntSlice>(&mut self, other: &S) -> &mut Self {
-        assert_eq!(self.len(), other.len(), "IntSlice length mismatch");
-        // Snapshot both overflow sets (O(k), tiny) before mutating self.
-        // 255 = +∞ on u8, so byte-level min is correct in all cases except
-        // both-overflow: only those slots need a fixup pass.
-        let self_ov: Vec<(usize, u32)> = self.overflow_entries().collect();
-        let other_ov: HashMap<usize, u32> = other.overflow_entries().collect();
-        self.clear_overflow();
-        // Pass 1 — SIMD-vectorizable byte min over the full primary array.
-        for (a, &b) in self.primary_bytes_mut().iter_mut().zip(other.primary_bytes()) {
-            if b < *a { *a = b; }
-        }
-        // Pass 2 — fixup slots where BOTH sides were overflow (primary = 255 after pass 1,
-        // but the overflow value may have changed).  Slots where only self was overflow are
-        // already correct: pass 1 wrote other.primary[slot] < 255 and clear_overflow removed
-        // the stale entry.
-        for (slot, self_val) in self_ov {
-            if let Some(&other_val) = other_ov.get(&slot) {
-                self.set(slot, self_val.min(other_val));
-            }
-        }
-        self
-    }
-
-    fn max<S: IntSlice>(&mut self, other: &S) -> &mut Self {
-        assert_eq!(self.len(), other.len(), "IntSlice length mismatch");
-        // Pre-pass — process other's overflow entries BEFORE the byte pass.
-        // After the byte pass, self.primary[slot] = 255 for all slots in other_ov,
-        // making it impossible to recover the original self value; we need it now.
-        for (slot, other_val) in other.overflow_entries() {
-            let self_val = self.get(slot);
-            self.set(slot, self_val.max(other_val));
-        }
-        // Pass 1 — SIMD-vectorizable byte max over the full primary array.
-        // 255 = +∞ on u8 → max(a, 255) = 255 is the correct sentinel for all
-        // overflow slots, whether handled by the pre-pass or already in self.
-        for (a, &b) in self.primary_bytes_mut().iter_mut().zip(other.primary_bytes()) {
-            if b > *a { *a = b; }
-        }
-        self
-    }
-
-    fn add<S: IntSlice>(&mut self, other: &S) -> &mut Self {
-        assert_eq!(self.len(), other.len(), "IntSlice length mismatch");
-        let n = self.len();
-        for s in 0..n {
-            // Read both primary bytes first — u8 is Copy, borrows released immediately.
-            let sb = self.primary_bytes()[s];
-            let ob = other.primary_bytes()[s];
-            if sb < 255 && ob < 255 {
-                // Hot path: no overflow lookup, no HashMap write in the common case.
-                let sum = sb as u32 + ob as u32;
-                if sum < 255 { self.primary_bytes_mut()[s] = sum as u8; }
-                else         { self.set(s, sum); }
-            } else {
-                // At least one side is in overflow — get() is unavoidable.
-                let self_val = self.get(s);
-                let other_val = other.get(s);
-                self.set(s, self_val + other_val);
-            }
-        }
-        self
-    }
-
-    fn diff<S: IntSlice>(&mut self, other: &S) -> &mut Self {
-        assert_eq!(self.len(), other.len(), "IntSlice length mismatch");
-        let n = self.len();
-        for s in 0..n {
-            let sb = self.primary_bytes()[s];
-            let ob = other.primary_bytes()[s];
-            if sb < 255 {
-                // Result is always < 255 — no overflow created or consulted.
-                // ob == 255 means b ≥ 255 > a, so saturating result = 0.
-                self.primary_bytes_mut()[s] = if ob < 255 { sb.saturating_sub(ob) } else { 0 };
-            } else {
-                // sb == 255: self has overflow — get() unavoidable.
-                // other.get() only needed when ob == 255 too (both-overflow case).
-                let self_val = self.get(s);
-                let other_val = if ob < 255 { ob as u32 } else { other.get(s) };
-                self.set(s, self_val.saturating_sub(other_val));
-            }
-        }
-        self
-    }
-
-    /// For each slot where `bits` is true, increment `self` by 1.
-    /// Skips zero words entirely — O(n_ones) rather than O(n).
-    fn count_bits<B: BitSlice>(&mut self, bits: &B) -> &mut Self {
-        assert_eq!(self.len(), bits.len(), "IntSlice/BitSlice length mismatch");
-        for (w_idx, &word) in bits.words().iter().enumerate() {
-            if word == 0 { continue; }
-            let base = w_idx * 64;
-            let mut w = word;
-            while w != 0 {
-                let bit = w.trailing_zeros() as usize;
-                let slot = base + bit;
-                if slot < self.len() { self.inc(slot); }
-                w &= w - 1;
-            }
-        }
-        self
-    }
-
-    /// Zero every slot where the corresponding bit in `mask` is 0.
-    /// Iterates only the zero bits — O(n_zeros), O(1) when mask is all-ones.
-    fn mask_with<B: BitSlice>(&mut self, mask: &B) -> &mut Self {
-        assert_eq!(self.len(), mask.len(), "IntSlice/BitSlice length mismatch");
-        let n = self.len();
-        for (wi, &word) in mask.words().iter().enumerate() {
-            if word == u64::MAX { continue; }
-            let mut zeros = !word;
-            while zeros != 0 {
-                let bit = zeros.trailing_zeros() as usize;
-                let s   = wi * 64 + bit;
-                if s < n {
-                    // u8 is Copy — the immutable borrow from primary_bytes() ends
-                    // before the mutable borrow from set() begins.
-                    let b = self.primary_bytes()[s];
-                    if b != 0 { self.set(s, 0); }
-                }
-                zeros &= zeros - 1;
-            }
-        }
-        self
-    }
-}
-
-// ── IntSlice → MemoryBitVec conversions ───────────────────────────────────────
-
-use crate::memoryvec::MemoryBitVec;
-
-pub trait IntToBit: IntSlice {
-    /// Bit set iff value >= threshold. Consistent with `geq` and `build_from_counts`.
-    fn to_bitvec(&self, threshold: u32) -> MemoryBitVec { self.geq(threshold) }
-
-    /// Bit set iff value >= 1 (slot is present).
-    fn to_presence(&self) -> MemoryBitVec { self.geq(1) }
-}
-
-impl<T: IntSlice> IntToBit for T {}
-
-// ── BitSlice → MemoryIntVec conversion ───────────────────────────────────────
-
-use crate::memoryintvec::MemoryIntVec;
-
-// Maps each byte value to its 8 constituent bits as individual u8 (0 or 1).
-static EXPAND_BYTE: [[u8; 8]; 256] = {
-    let mut table = [[0u8; 8]; 256];
-    let mut b = 0usize;
-    while b < 256 {
-        let mut bit = 0usize;
-        while bit < 8 {
-            table[b][bit] = ((b >> bit) & 1) as u8;
-            bit += 1;
-        }
-        b += 1;
-    }
-    table
-};
-
-pub trait BitToInt: BitSlice {
-    fn to_intvec(&self) -> MemoryIntVec {
-        let n = self.len();
-        let mut primary = vec![0u8; n];
-
-        let words = self.words();
-        let full_words = n / 64;
-
-        for (w_idx, &word) in words[..full_words].iter().enumerate() {
-            let base = w_idx * 64;
-            for byte_off in 0..8usize {
-                let byte = (word >> (byte_off * 8)) as u8;
-                primary[base + byte_off * 8..base + byte_off * 8 + 8]
-                    .copy_from_slice(&EXPAND_BYTE[byte as usize]);
-            }
-        }
-
-        let rem = n % 64;
-        if rem > 0 {
-            let word = words[full_words];
-            let base = full_words * 64;
-            for bit in 0..rem {
-                primary[base + bit] = ((word >> bit) & 1) as u8;
-            }
-        }
-
-        MemoryIntVec::from_primary(primary)
-    }
-}
-
-impl<T: BitSlice> BitToInt for T {}
-
 // ── Column-level weight statistic — total count or presence count per column.
 /// Additive across layers and partitions; used as denominator in normalised distances.
 ///

src/obicompactvec/src/views.rs

@@ -0,0 +1,278 @@
+use crate::format::{byte_count_nonzero, byte_sum, parse_overflow_entry};
+
+// ── BitSliceView ──────────────────────────────────────────────────────────────
+
+/// Lightweight, copy-able read-only view over a u64 word array.
+/// Bit `i` is in `words[i >> 6]` at position `i & 63`.  Padding bits are zero.
+#[derive(Clone, Copy)]
+pub struct BitSliceView<'a> {
+    pub(crate) words: &'a [u64],
+    pub(crate) n:     usize,
+}
+
+impl<'a> BitSliceView<'a> {
+    #[inline]
+    pub fn new(words: &'a [u64], n: usize) -> Self { Self { words, n } }
+
+    pub fn len(&self)      -> usize  { self.n }
+    pub fn is_empty(&self) -> bool   { self.n == 0 }
+    pub fn words(&self)    -> &'a [u64] { self.words }
+
+    #[inline]
+    pub fn get(&self, slot: usize) -> bool {
+        (self.words[slot >> 6] >> (slot & 63)) & 1 != 0
+    }
+
+    pub fn count_ones(&self) -> u64 {
+        self.words.iter().map(|w| w.count_ones() as u64).sum()
+    }
+    pub fn count_zeros(&self) -> u64 { self.n as u64 - self.count_ones() }
+
+    pub fn iter(&self) -> BitSliceIter<'a> {
+        BitSliceIter { words: self.words, slot: 0, n: self.n }
+    }
+
+    pub fn partial_jaccard_dist(self, other: BitSliceView<'_>) -> (u64, u64) {
+        assert_eq!(self.n, other.n, "BitSliceView length mismatch");
+        self.words.iter().zip(other.words)
+            .fold((0u64, 0u64), |(i, u), (&a, &b)| {
+                (i + (a & b).count_ones() as u64, u + (a | b).count_ones() as u64)
+            })
+    }
+
+    pub fn jaccard_dist(self, other: BitSliceView<'_>) -> f64 {
+        let (inter, union) = self.partial_jaccard_dist(other);
+        if union == 0 { 0.0 } else { 1.0 - inter as f64 / union as f64 }
+    }
+
+    pub fn hamming_dist(self, other: BitSliceView<'_>) -> u64 {
+        assert_eq!(self.n, other.n, "BitSliceView length mismatch");
+        self.words.iter().zip(other.words)
+            .map(|(&a, &b)| (a ^ b).count_ones() as u64)
+            .sum()
+    }
+}
+
+// ── BitSliceIter ──────────────────────────────────────────────────────────────
+
+pub struct BitSliceIter<'a> {
+    words: &'a [u64],
+    slot:  usize,
+    n:     usize,
+}
+
+impl Iterator for BitSliceIter<'_> {
+    type Item = bool;
+    fn next(&mut self) -> Option<bool> {
+        if self.slot >= self.n { return None; }
+        let v = (self.words[self.slot >> 6] >> (self.slot & 63)) & 1 != 0;
+        self.slot += 1;
+        Some(v)
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let rem = self.n - self.slot;
+        (rem, Some(rem))
+    }
+}
+impl ExactSizeIterator for BitSliceIter<'_> {}
+
+// ── IntSliceView ──────────────────────────────────────────────────────────────
+
+/// Lightweight, copy-able read-only view over a compact-int primary array plus
+/// its sorted raw overflow bytes.  Zero-copy: all data lives in the caller's mmap.
+#[derive(Clone, Copy)]
+pub struct IntSliceView<'a> {
+    pub(crate) primary:      &'a [u8],
+    pub(crate) overflow_raw: &'a [u8],   // n_overflow × OVERFLOW_ENTRY_SIZE bytes, sorted by slot
+    pub(crate) n_overflow:   usize,
+    pub(crate) n:            usize,
+}
+
+impl<'a> IntSliceView<'a> {
+    #[inline]
+    pub fn new(primary: &'a [u8], overflow_raw: &'a [u8], n_overflow: usize, n: usize) -> Self {
+        Self { primary, overflow_raw, n_overflow, n }
+    }
+
+    pub fn len(&self)        -> usize    { self.n }
+    pub fn is_empty(&self)   -> bool     { self.n == 0 }
+    pub fn primary_bytes(&self) -> &'a [u8] { self.primary }
+    pub fn n_overflow(&self) -> usize    { self.n_overflow }
+
+    pub fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + 'a {
+        let raw  = self.overflow_raw;
+        let n_ov = self.n_overflow;
+        (0..n_ov).map(move |i| parse_overflow_entry(raw, 0, i))
+    }
+
+    /// O(log n_overflow) via binary search (overflow is always sorted by slot).
+    pub fn get(&self, slot: usize) -> u32 {
+        let b = self.primary[slot];
+        if b < 255 { return b as u32; }
+        let mut lo = 0usize;
+        let mut hi = self.n_overflow;
+        while lo < hi {
+            let mid = lo + (hi - lo) / 2;
+            let (s, v) = parse_overflow_entry(self.overflow_raw, 0, mid);
+            match s.cmp(&slot) {
+                std::cmp::Ordering::Equal   => return v,
+                std::cmp::Ordering::Less    => lo = mid + 1,
+                std::cmp::Ordering::Greater => hi = mid,
+            }
+        }
+        panic!("slot {slot} marked overflow but not found")
+    }
+
+    /// Sequential merge scan: yields all n values in slot order.
+    pub fn iter(&self) -> IntSliceViewIter<'a> {
+        IntSliceViewIter {
+            primary:      self.primary,
+            overflow_raw: self.overflow_raw,
+            slot:         0,
+            overflow_pos: 0,
+            n:            self.n,
+        }
+    }
+
+    pub fn sum(&self) -> u64 {
+        byte_sum(self.primary, self.overflow_entries().map(|(_, v)| v))
+    }
+
+    pub fn count_nonzero(&self) -> u64 {
+        byte_count_nonzero(self.primary)
+    }
+
+    // ── Distance methods ──────────────────────────────────────────────────────
+
+    pub fn partial_bray_dist(self, other: IntSliceView<'_>) -> u64 {
+        assert_eq!(self.n, other.n, "length mismatch");
+        self.iter().zip(other.iter()).map(|(a, b)| a.min(b) as u64).sum()
+    }
+
+    pub fn bray_dist(self, other: IntSliceView<'_>) -> f64 {
+        let sum_min = self.partial_bray_dist(other);
+        let denom = self.sum() + other.sum();
+        if denom == 0 { 0.0 } else { 1.0 - 2.0 * sum_min as f64 / denom as f64 }
+    }
+
+    pub fn partial_relfreq_bray_dist(self, other: IntSliceView<'_>, sa: f64, sb: f64) -> f64 {
+        assert_eq!(self.n, other.n, "length mismatch");
+        self.iter().zip(other.iter())
+            .map(|(a, b)| {
+                let pa = if sa > 0.0 { a as f64 / sa } else { 0.0 };
+                let pb = if sb > 0.0 { b as f64 / sb } else { 0.0 };
+                pa.min(pb)
+            })
+            .sum()
+    }
+
+    pub fn relfreq_bray_dist(self, other: IntSliceView<'_>) -> f64 {
+        let sa = self.sum() as f64;
+        let sb = other.sum() as f64;
+        if sa == 0.0 && sb == 0.0 { return 0.0; }
+        1.0 - self.partial_relfreq_bray_dist(other, sa, sb)
+    }
+
+    pub fn partial_euclidean_dist(self, other: IntSliceView<'_>) -> f64 {
+        assert_eq!(self.n, other.n, "length mismatch");
+        self.iter().zip(other.iter())
+            .map(|(a, b)| { let d = a as f64 - b as f64; d * d })
+            .sum()
+    }
+
+    pub fn euclidean_dist(self, other: IntSliceView<'_>) -> f64 {
+        self.partial_euclidean_dist(other).sqrt()
+    }
+
+    pub fn partial_relfreq_euclidean_dist(self, other: IntSliceView<'_>, sa: f64, sb: f64) -> f64 {
+        assert_eq!(self.n, other.n, "length mismatch");
+        self.iter().zip(other.iter())
+            .map(|(a, b)| {
+                let pa = if sa > 0.0 { a as f64 / sa } else { 0.0 };
+                let pb = if sb > 0.0 { b as f64 / sb } else { 0.0 };
+                let d = pa - pb;
+                d * d
+            })
+            .sum()
+    }
+
+    pub fn relfreq_euclidean_dist(self, other: IntSliceView<'_>) -> f64 {
+        let sa = self.sum() as f64;
+        let sb = other.sum() as f64;
+        if sa == 0.0 && sb == 0.0 { return 0.0; }
+        self.partial_relfreq_euclidean_dist(other, sa, sb).sqrt()
+    }
+
+    pub fn partial_hellinger_euclidean_dist(self, other: IntSliceView<'_>, sa: f64, sb: f64) -> f64 {
+        assert_eq!(self.n, other.n, "length mismatch");
+        self.iter().zip(other.iter())
+            .map(|(a, b)| {
+                let pa = if sa > 0.0 { (a as f64 / sa).sqrt() } else { 0.0 };
+                let pb = if sb > 0.0 { (b as f64 / sb).sqrt() } else { 0.0 };
+                let d = pa - pb;
+                d * d
+            })
+            .sum()
+    }
+
+    pub fn hellinger_euclidean_dist(self, other: IntSliceView<'_>) -> f64 {
+        let sa = self.sum() as f64;
+        let sb = other.sum() as f64;
+        if sa == 0.0 && sb == 0.0 { return 0.0; }
+        self.partial_hellinger_euclidean_dist(other, sa, sb).sqrt()
+    }
+
+    pub fn hellinger_dist(self, other: IntSliceView<'_>) -> f64 {
+        self.hellinger_euclidean_dist(other) / std::f64::consts::SQRT_2
+    }
+
+    pub fn partial_threshold_jaccard_dist(self, other: IntSliceView<'_>, threshold: u32) -> (u64, u64) {
+        assert_eq!(self.n, other.n, "length mismatch");
+        self.iter().zip(other.iter())
+            .fold((0u64, 0u64), |(inter, uni), (a, b)| {
+                let ap = a >= threshold;
+                let bp = b >= threshold;
+                (inter + (ap & bp) as u64, uni + (ap | bp) as u64)
+            })
+    }
+
+    pub fn threshold_jaccard_dist(self, other: IntSliceView<'_>, threshold: u32) -> f64 {
+        let (inter, union) = self.partial_threshold_jaccard_dist(other, threshold);
+        if union == 0 { 0.0 } else { 1.0 - inter as f64 / union as f64 }
+    }
+
+    pub fn jaccard_dist(self, other: IntSliceView<'_>) -> f64 {
+        self.threshold_jaccard_dist(other, 1)
+    }
+}
+
+// ── IntSliceViewIter ──────────────────────────────────────────────────────────
+
+pub struct IntSliceViewIter<'a> {
+    primary:      &'a [u8],
+    overflow_raw: &'a [u8],
+    slot:         usize,
+    overflow_pos: usize,
+    n:            usize,
+}
+
+impl Iterator for IntSliceViewIter<'_> {
+    type Item = u32;
+    fn next(&mut self) -> Option<u32> {
+        if self.slot >= self.n { return None; }
+        let v = self.primary[self.slot];
+        self.slot += 1;
+        if v < 255 {
+            Some(v as u32)
+        } else {
+            let (_, val) = parse_overflow_entry(self.overflow_raw, 0, self.overflow_pos);
+            self.overflow_pos += 1;
+            Some(val)
+        }
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let rem = self.n - self.slot;
+        (rem, Some(rem))
+    }
+}
+impl ExactSizeIterator for IntSliceViewIter<'_> {}

src/obidebruinj/src/debruijn.rs

@@ -3,6 +3,7 @@ use crossbeam_channel;
 use hashbrown::HashMap;
 use obikseq::k;
 use obikseq::{CanonicalKmer, Sequence, Unitig};
+#[cfg(not(any(test, feature = "test-utils")))]
 use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
 use std::cell::RefCell;
 use std::fmt;

src/obikindex/Cargo.toml

@@ -17,4 +17,8 @@ serde            = { version = "1", features = ["derive"] }
 serde_json       = "1"
 indicatif        = "0.17"
 tracing          = "0.1.44"
-hwlocality       = { version = "1.0.0-alpha.11", features = ["vendored"] }
+hwlocality       = { version = "1.0.0-alpha.11", features = ["vendored"], optional = true }
+
+[features]
+default = ["numa"]
+numa    = ["hwlocality"]

src/obikindex/src/merge.rs

@@ -11,7 +11,7 @@ use obilayeredmap::IndexMode;
 use crate::error::{OKIError, OKIResult};
 use crate::index::KmerIndex;
 use crate::meta::{GenomeInfo, IndexMeta};
-use crate::state::IndexState;
+use crate::state::{IndexState, SENTINEL_INDEXED};
 
 pub use obikpartitionner::MergeMode;
 
@@ -263,6 +263,8 @@ impl KmerIndex {
             rep.push(t.stop());
         }
 
+        fs::File::create(output.join(SENTINEL_INDEXED)).map_err(OKIError::Io)?;
+
         KmerIndex::open(output)
     }
 }

src/obikindex/src/numa.rs

@@ -5,79 +5,109 @@
 // CPUs. Linux first-touch policy then places graph allocations in local DRAM
 // automatically — no explicit memory binding needed.
 //
-// Returns None when:
-//   - hwloc topology initialisation fails
-//   - the system has only one NUMA node (UMA, Apple Silicon, single-socket)
-//   - any per-node pool fails to build
+// UMA systems (single socket, Apple Silicon, etc.) are the degenerate case:
+// one synthetic node containing all cores, no pool, no pinning.
 
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use crossbeam_channel::unbounded;
+#[cfg(feature = "numa")]
 use hwlocality::Topology;
+#[cfg(feature = "numa")]
 use hwlocality::cpu::binding::CpuBindingFlags;
+#[cfg(feature = "numa")]
 use hwlocality::cpu::cpuset::CpuSet;
+#[cfg(feature = "numa")]
 use hwlocality::object::types::ObjectType;
+use obisys::{CpuSample, IoSample};
 use tracing::debug;
 
 // ── Public interface ──────────────────────────────────────────────────────────
 
 pub struct NumaSetup {
-    pub pools: Vec<Arc<rayon::ThreadPool>>,
+    /// One entry per NUMA node. `None` on UMA systems (no pool, no pinning).
+    pub pools: Vec<Option<Arc<rayon::ThreadPool>>>,
     /// CPU indices for each NUMA node, in node order.
     pub cpus_per_node: Vec<Vec<usize>>,
 }
 
 impl NumaSetup {
-    /// Workers to activate per NUMA node.
-    /// Empirically ~3 workers saturate one node's memory bandwidth.
+    /// Maximum worker slots per node (one per physical core in the node).
     pub fn workers_per_node(&self) -> usize {
         self.cpus_per_node
             .first()
-            .map(|c| (c.len() / 8).max(3).min(8))
-            .unwrap_or(3)
+            .map(|c| c.len().max(1))
+            .unwrap_or(1)
     }
 }
 
 /// Detect NUMA topology and build per-node Rayon pools.
-/// Returns None on UMA systems, single-node machines, or on failure.
-pub fn build() -> Option<NumaSetup> {
-    let topology = Topology::new().ok()?;
+/// Always succeeds: falls back to a single synthetic UMA node on failure.
+#[cfg(feature = "numa")]
+pub fn build() -> NumaSetup {
+    if let Ok(topology) = Topology::new() {
+        let nodes: Vec<Vec<usize>> = topology
+            .objects_with_type(ObjectType::NUMANode)
+            .filter_map(|obj| obj.cpuset())
+            .map(|cpuset| {
+                cpuset
+                    .iter_set()
+                    .map(|idx| usize::from(idx))
+                    .collect::<Vec<_>>()
+            })
+            .filter(|v| !v.is_empty())
+            .collect();
 
-    let nodes: Vec<Vec<usize>> = topology
-        .objects_with_type(ObjectType::NUMANode)
-        .filter_map(|obj| obj.cpuset())
-        .map(|cpuset| {
-            cpuset
-                .iter_set()
-                .map(|idx| usize::from(idx))
-                .collect::<Vec<_>>()
-        })
-        .filter(|v| !v.is_empty())
-        .collect();
-
-    if nodes.len() <= 1 {
-        return None;
+        if nodes.len() > 1 {
+            if let Some(pools) = nodes
+                .iter()
+                .map(|cpus| build_pool(cpus).map(|p| Some(Arc::new(p))))
+                .collect::<Option<Vec<_>>>()
+            {
+                debug!(
+                    "NUMA topology: {} node(s), {} core(s)/node",
+                    nodes.len(),
+                    nodes.first().map_or(0, |v| v.len()),
+                );
+                return NumaSetup {
+                    pools,
+                    cpus_per_node: nodes,
+                };
+            }
+        }
     }
 
-    debug!(
-        "NUMA topology: {} node(s), {} core(s)/node",
-        nodes.len(),
-        nodes.first().map_or(0, |v| v.len()),
-    );
+    // UMA fallback: single synthetic node, all cores, no pool, no pinning.
+    let n_cores = std::thread::available_parallelism()
+        .map(|n| n.get())
+        .unwrap_or(1);
+    debug!("UMA: single synthetic node, {} core(s)", n_cores);
+    NumaSetup {
+        pools: vec![None],
+        cpus_per_node: vec![(0..n_cores).collect()],
+    }
+}
 
-    let pools = nodes
-        .iter()
-        .map(|cpus| build_pool(cpus).map(Arc::new))
-        .collect::<Option<Vec<_>>>()?;
-
-    Some(NumaSetup { pools, cpus_per_node: nodes })
+#[cfg(not(feature = "numa"))]
+pub fn build() -> NumaSetup {
+    let n_cores = std::thread::available_parallelism()
+        .map(|n| n.get())
+        .unwrap_or(1);
+    debug!("UMA: single synthetic node, {} core(s)", n_cores);
+    NumaSetup {
+        pools: vec![None],
+        cpus_per_node: vec![(0..n_cores).collect()],
+    }
 }
 
 /// Bind the calling thread to `cpu_indices` using hwloc.
 /// Silently returns on any error so the thread still runs, just unbound.
+#[cfg(feature = "numa")]
 pub fn pin_current_thread(cpu_indices: &[usize]) {
-    let Ok(topology) = Topology::new() else { return };
+    let Ok(topology) = Topology::new() else {
+        return;
+    };
     let mut cpuset = CpuSet::new();
     for &idx in cpu_indices {
         cpuset.set(idx);
@@ -85,8 +115,12 @@ pub fn pin_current_thread(cpu_indices: &[usize]) {
     let _ = topology.bind_cpu(&cpuset, CpuBindingFlags::THREAD);
 }
 
+#[cfg(not(feature = "numa"))]
+pub fn pin_current_thread(_cpu_indices: &[usize]) {}
+
 // ── Internal helpers ──────────────────────────────────────────────────────────
 
+#[cfg(feature = "numa")]
 fn build_pool(cpus: &[usize]) -> Option<rayon::ThreadPool> {
     let cpus = cpus.to_vec();
     rayon::ThreadPoolBuilder::new()
@@ -103,29 +137,49 @@ fn build_pool(cpus: &[usize]) -> Option<rayon::ThreadPool> {
         .ok()
 }
 
-// ── PartitionRunner ───────────────────────────────────────────────────────────
+// ── PartitionRunner ─────────────────────────────────────────────────────────
+
+/// Growth step (fraction of a node's worker capacity added per activation
+/// event, see [`NodeActivation::grow`]).
+const GROWTH_DIVISOR: usize = 8;
+/// Minimum CPU efficiency growth to activate more workers, as a fraction of
+/// the size of the *last growth step* (e.g. `0.2` after adding 8 workers
+/// requires the next check to show at least +1.6 cores of growth — 20 % of
+/// the ~8 cores those 8 workers should contribute if the workload is truly
+/// CPU-bound). Scaling by the last step's size — not the cumulative total —
+/// keeps the bar meaningful regardless of how many workers are already
+/// active, instead of demanding an ever-larger absolute jump as the pool
+/// grows.
+const CPU_SPAWN_THRESHOLD: f64 = 0.2;
+/// Minimum I/O throughput growth (relative) to activate more workers.
+const IO_SPAWN_THRESHOLD: f64 = 0.2;
 
 struct NodeConfig {
-    pool:        Option<Arc<rayon::ThreadPool>>,
-    cpu_ids:     Vec<usize>,
+    pool: Option<Arc<rayon::ThreadPool>>,
+    cpu_ids: Vec<usize>,
     max_workers: usize,
 }
 
 /// Generic NUMA-aware runner for partition-level parallel work.
 ///
-/// Workers are distributed round-robin across NUMA nodes and pinned to their
-/// node's CPUs.  UMA systems are the degenerate case: one node, no pinning.
+/// Workers are distributed evenly across NUMA nodes and pinned to their
+/// node's CPUs. UMA is the degenerate case: one node, no pinning.
+///
+/// Workers are pre-spawned dormant, one activation channel per node so
+/// growth always targets a specific node rather than whichever dormant
+/// worker happens to wake up first on a shared channel. Growth (both the
+/// initial count and each subsequent step) is expressed as a fraction of
+/// `workers_per_node`, applied identically to every node, so the pace of
+/// ramp-up depends on node size rather than node count — a single-NUMA-node
+/// (UMA) machine ramps just as fast as an 8-node one.
 ///
 /// # Termination
 ///
-/// Termination is driven entirely by channel closure:
-///
 /// ```text
-/// drop(part_tx)    → part_rx drains → workers exit → drop their result_tx
-/// drop(result_tx)  → result_rx closes → controller loop exits
+/// drop(part_tx)      → part_rx drains → workers exit → drop their result_tx
+/// drop(result_tx)    → result_rx closes → controller loop exits
+/// drop(activate_txs) → dormant workers exit cleanly
 /// ```
-///
-/// No explicit counter or sentinel needed.
 pub struct PartitionRunner {
     nodes: Vec<NodeConfig>,
 }
@@ -136,116 +190,308 @@ impl PartitionRunner {
         self.nodes.iter().map(|n| n.max_workers).sum()
     }
 
-    /// Detect topology and build.  Falls back to a single-node UMA runner on
-    /// macOS, single-socket machines, or hwloc failure.
+    /// Detect topology and build. Always succeeds.
     pub fn new() -> Self {
-        match build() {
-            Some(ns) => {
-                let wpn = ns.workers_per_node();
-                debug!(
-                    "PartitionRunner: NUMA mode — {} node(s) × {} worker(s)/node",
-                    ns.pools.len(), wpn,
-                );
-                let nodes = ns.pools
-                    .into_iter()
-                    .zip(ns.cpus_per_node)
-                    .map(|(pool, cpu_ids)| NodeConfig {
-                        pool:        Some(pool),
-                        cpu_ids,
-                        max_workers: wpn,
-                    })
-                    .collect();
-                Self { nodes }
-            }
-            None => {
-                let n_cores = std::thread::available_parallelism()
-                    .map(|n| n.get())
-                    .unwrap_or(1);
-                let max_workers = (n_cores / 2).max(1);
-                debug!("PartitionRunner: UMA mode — {} worker(s)", max_workers);
-                Self {
-                    nodes: vec![NodeConfig {
-                        pool:        None,
-                        cpu_ids:     vec![],
-                        max_workers,
-                    }],
-                }
-            }
-        }
+        let ns = build();
+        let wpn = ns.workers_per_node();
+        debug!(
+            "PartitionRunner: {} node(s) × {} worker(s)/node max",
+            ns.pools.len(),
+            wpn,
+        );
+        let nodes = ns
+            .pools
+            .into_iter()
+            .zip(ns.cpus_per_node)
+            .map(|(pool, cpu_ids)| NodeConfig {
+                pool,
+                cpu_ids,
+                max_workers: wpn,
+            })
+            .collect();
+        Self { nodes }
     }
 
     /// Run `f(i)` for every index in `order`.
     ///
-    /// Workers are spawned upfront and distributed round-robin across NUMA
-    /// nodes.  `on_done(i, result, elapsed)` is called from the controller
-    /// thread as each partition completes — suitable for progress bars and
-    /// result aggregation.
+    /// Workers are pre-spawned dormant and activated adaptively, per node:
+    /// `(workers_per_node / INITIAL_DIVISOR).max(1)` are woken immediately on
+    /// every node, then `(workers_per_node / GROWTH_DIVISOR).max(1)` more per
+    /// node each time the check below fires. A timer thread fires that check
+    /// every `TIMER_SECS` seconds; each completed partition resets that timer
+    /// (forcing an immediate check) and also triggers its own inline check. A
+    /// growth step happens whenever CPU efficiency grows by at least
+    /// `CPU_SPAWN_THRESHOLD` of what the last growth step should have
+    /// contributed, or I/O throughput grows by at least `IO_SPAWN_THRESHOLD`
+    /// (relative) since the last check — whichever resource is the actual
+    /// bottleneck still shows headroom.
+    ///
+    /// `on_done(i, result, elapsed)` is called from the controller thread as
+    /// each partition completes — suitable for progress bars and result
+    /// aggregation.
     ///
     /// Returns the first error produced by `f`, if any.
-    pub fn run<F, R, E, C>(
-        &self,
-        order:      &[usize],
-        f:          F,
-        mut on_done: C,
-    ) -> Result<(), E>
+    pub fn run<F, R, E, C>(&self, order: &[usize], f: F, mut on_done: C) -> Result<(), E>
     where
         F: Fn(usize) -> Result<R, E> + Send + Sync,
         R: Send,
         E: Send,
         C: FnMut(usize, R, Duration) + Send,
     {
-        // Pre-load the work queue, then drop the sender so workers' part_rx
-        // iterators terminate when the queue is drained.
+        let n_total = order.len();
+        if n_total == 0 {
+            return Ok(());
+        }
+
+        const TIMER_SECS: u64 = 30;
+        const INITIAL_DIVISOR: usize = 4;
+
+        // ── Channels ──────────────────────────────────────────────────────────
         let (part_tx, part_rx) = unbounded::<usize>();
-        for &i in order { part_tx.send(i).ok(); }
+        // reset_tx: controller → timer ("reset the 30 s window")
+        let (reset_tx, reset_rx) = unbounded::<()>();
+        // event_tx: workers + timer → controller (unified event stream)
+        let (event_tx, event_rx) = unbounded::<WorkerEvent<R, E>>();
+        // One activation channel per node: growth always targets a specific
+        // node, rather than whichever dormant worker happens to win the race
+        // on a channel shared across all nodes.
+        let (activate_txs, activate_rxs): (Vec<_>, Vec<_>) =
+            (0..self.nodes.len()).map(|_| unbounded::<()>()).unzip();
+
+        for &i in order {
+            part_tx.send(i).ok();
+        }
         drop(part_tx);
 
-        let (result_tx, result_rx) = unbounded::<(usize, Result<R, E>, Duration)>();
-        let n_nodes = self.nodes.len();
-        let f = &f; // shared borrow; F: Sync so concurrent calls are safe
+        let max_workers = self.max_workers();
+        let node_caps: Vec<usize> = self.nodes.iter().map(|n| n.max_workers).collect();
+        let f = &f;
 
         let mut first_err: Option<E> = None;
 
         std::thread::scope(|s| {
-            // Spawn all workers upfront, round-robin across NUMA nodes.
-            for w in 0..self.max_workers() {
-                let node    = &self.nodes[w % n_nodes];
-                let prx     = part_rx.clone();
-                let rtx     = result_tx.clone();
-                let pool    = node.pool.clone();
-                let cpu_ids = &node.cpu_ids;
-
-                s.spawn(move || {
-                    if !cpu_ids.is_empty() { pin_current_thread(cpu_ids); }
-                    for i in &prx {
-                        let t = Instant::now();
-                        let r = match &pool {
-                            Some(p) => p.install(|| f(i)),
-                            None    => f(i),
-                        };
-                        rtx.send((i, r, t.elapsed())).ok();
+            // ── Timer thread ──────────────────────────────────────────────────
+            // Sends TimerTick every TIMER_SECS seconds. Resets its window each
+            // time reset_rx receives a message (i.e. on partition completion).
+            let timer_tx = event_tx.clone();
+            s.spawn(move || {
+                let period = Duration::from_secs(TIMER_SECS);
+                loop {
+                    crossbeam_channel::select! {
+                        recv(reset_rx) -> r => {
+                            if r.is_err() { break; } // reset_tx dropped → exit
+                        }
+                        default(period) => {
+                            if timer_tx.send(WorkerEvent::TimerTick).is_err() { break; }
+                        }
                     }
-                });
-            }
+                }
+            });
 
-            // Drop the controller's sender: result_rx closes once all worker
-            // rtx clones are dropped (i.e. all workers have exited).
-            drop(result_tx);
+            // ── Pre-spawn workers dormant, grouped by node ────────────────────
+            // Each worker listens on its own node's activation channel only.
+            for (node, arx) in self.nodes.iter().zip(activate_rxs.iter()) {
+                let cpu_ids = &node.cpu_ids;
+                for _ in 0..node.max_workers {
+                    let prx = part_rx.clone();
+                    let etx = event_tx.clone();
+                    let arx = arx.clone();
+                    let pool = node.pool.clone();
 
-            // Drain results concurrently with workers.  The for loop exits
-            // when result_rx is disconnected — at that point all workers are
-            // done and the scope join below is instantaneous.
-            for (i, r, dur) in &result_rx {
-                match r {
-                    Ok(v)  => on_done(i, v, dur),
-                    Err(e) => { if first_err.is_none() { first_err = Some(e); } }
+                    s.spawn(move || {
+                        if arx.recv().is_err() {
+                            return;
+                        }
+                        if !cpu_ids.is_empty() {
+                            pin_current_thread(cpu_ids);
+                        }
+                        for i in &prx {
+                            let t = Instant::now();
+                            let r = match &pool {
+                                Some(p) => p.install(|| f(i)),
+                                None => f(i),
+                            };
+                            etx.send(WorkerEvent::Completed(i, r, t.elapsed())).ok();
+                        }
+                    });
                 }
             }
+            // Drop controller's event_tx: event_rx closes when all workers +
+            // timer have exited.
+            drop(event_tx);
+
+            // ── Controller ────────────────────────────────────────────────────
+            let mut activation = NodeActivation::new(&activate_txs, &node_caps, max_workers);
+            activation.activate_initial(INITIAL_DIVISOR, n_total);
+
+            let mut cpu_sample = CpuSample::now();
+            let mut io_sample = IoSample::now();
+            let mut completed = 0usize;
+
+            while completed < n_total {
+                let Ok(event) = event_rx.recv() else { break };
+                match event {
+                    WorkerEvent::Completed(i, r, dur) => {
+                        match r {
+                            Ok(v) => on_done(i, v, dur),
+                            Err(e) => {
+                                if first_err.is_none() {
+                                    first_err = Some(e);
+                                }
+                            }
+                        }
+                        completed += 1;
+                        // Reset the 30 s timer.
+                        reset_tx.send(()).ok();
+                        // Inline check: same logic as a timer tick.
+                        maybe_activate(
+                            &mut activation,
+                            &mut cpu_sample,
+                            &mut io_sample,
+                            completed,
+                            n_total,
+                        );
+                    }
+                    WorkerEvent::TimerTick => {
+                        maybe_activate(
+                            &mut activation,
+                            &mut cpu_sample,
+                            &mut io_sample,
+                            completed,
+                            n_total,
+                        );
+                    }
+                }
+            }
+
+            // Dormant workers exit once every sender for their node's channel
+            // is dropped — `activate_txs` holds the only ones.
+            drop(activate_txs);
+            // Timer thread exits when reset_tx closes.
+            drop(reset_tx);
         });
 
         match first_err {
             Some(e) => Err(e),
-            None    => Ok(()),
+            None => Ok(()),
         }
     }
 }
+
+// ── Internal event type ───────────────────────────────────────────────────────
+
+enum WorkerEvent<R, E> {
+    Completed(usize, Result<R, E>, Duration),
+    TimerTick,
+}
+
+/// Tracks how many of each node's dormant workers have been woken, and
+/// grows every node by the same amount at each step (capped by that node's
+/// remaining dormant workers and by the run's total budget) so load stays
+/// balanced across nodes at every point in time — never just "one more
+/// worker somewhere". Also remembers the size of the last real growth step
+/// (`last_step`), used to scale the CPU activation threshold to what that
+/// step could plausibly have contributed (see `maybe_activate`).
+struct NodeActivation<'a> {
+    txs: &'a [crossbeam_channel::Sender<()>],
+    caps: &'a [usize],
+    active: Vec<usize>,
+    total: usize,
+    max: usize,
+    last_step: usize,
+}
+
+impl<'a> NodeActivation<'a> {
+    fn new(txs: &'a [crossbeam_channel::Sender<()>], caps: &'a [usize], max: usize) -> Self {
+        Self {
+            txs,
+            caps,
+            active: vec![0; txs.len()],
+            total: 0,
+            max,
+            last_step: 0,
+        }
+    }
+
+    fn total(&self) -> usize {
+        self.total
+    }
+    fn last_step(&self) -> usize {
+        self.last_step
+    }
+    fn max(&self) -> usize {
+        self.max
+    }
+    fn is_full(&self) -> bool {
+        self.total >= self.max
+    }
+
+    /// Wake up to `(node_cap / divisor).max(1)` dormant workers on every
+    /// node, capped by `n_total`. Called once at startup, unconditionally.
+    fn activate_initial(&mut self, divisor: usize, n_total: usize) {
+        self.grow(divisor, n_total);
+    }
+
+    /// Same per-node sizing as [`activate_initial`](Self::activate_initial),
+    /// applied as a growth step. Returns the number of workers actually
+    /// activated (may be less than requested once a node or the total
+    /// budget is exhausted). Updates `last_step` when it actually grew.
+    fn grow(&mut self, divisor: usize, n_total: usize) -> usize {
+        let before = self.total;
+        for idx in 0..self.txs.len() {
+            let wanted = (self.caps[idx] / divisor).max(1);
+            let room = self.caps[idx].saturating_sub(self.active[idx]);
+            let grow = wanted.min(room).min(n_total.saturating_sub(self.total));
+            for _ in 0..grow {
+                self.txs[idx].send(()).ok();
+            }
+            self.active[idx] += grow;
+            self.total += grow;
+        }
+        let grew = self.total - before;
+        if grew > 0 {
+            self.last_step = grew;
+        }
+        grew
+    }
+}
+
+fn maybe_activate(
+    activation: &mut NodeActivation,
+    cpu_sample: &mut CpuSample,
+    io_sample: &mut IoSample,
+    completed: usize,
+    n_total: usize,
+) {
+    if activation.is_full() || completed >= n_total {
+        return;
+    }
+
+    // Expect roughly 1 core of extra efficiency per worker activated in the
+    // last growth step (CPU-bound case); require at least CPU_SPAWN_THRESHOLD
+    // (20 %) of that expected gain before growing again. Scaling by the last
+    // step's size — not the cumulative total — keeps the bar meaningful
+    // regardless of how many workers are already active: growing by 8 should
+    // always take ~+1.6 cores to confirm, whether that's the 2nd growth step
+    // or the 20th.
+    let cpu_threshold = CPU_SPAWN_THRESHOLD * activation.last_step() as f64;
+
+    // Call both unconditionally (no `||` short-circuit): each sampler must
+    // advance its own window every tick, regardless of what the other one
+    // reports, or it would starve behind whichever signal fires first.
+    let cpu_wants_more = cpu_sample.do_i_activate(cpu_threshold);
+    let io_wants_more = io_sample.do_i_activate(IO_SPAWN_THRESHOLD * activation.last_step() as f64);
+    if !(cpu_wants_more || io_wants_more) {
+        return;
+    }
+
+    let grew = activation.grow(GROWTH_DIVISOR, n_total);
+    if grew > 0 {
+        debug!(
+            "activated {} worker(s) — {}/{} active",
+            grew,
+            activation.total(),
+            activation.max()
+        );
+    }
+}

src/obikindex/src/select.rs

@@ -3,7 +3,7 @@ use std::io;
 use std::path::Path;
 
 use obikpartitionner::{KmerPartition, OutputCol, PARTITIONS_SUBDIR};
-use obisys::{Stage, progress_bar};
+use obisys::{Reporter, Stage, progress_bar};
 use tracing::info;
 
 use crate::error::{OKIError, OKIResult};
@@ -25,6 +25,7 @@ impl KmerIndex {
         threshold: u32,
         output_presence: bool,
         force: bool,
+        rep: &mut Reporter,
     ) -> OKIResult<Self> {
         let output = output.as_ref();
 
@@ -80,13 +81,14 @@ impl KmerIndex {
         ).map_err(OKIError::Partition)?;
 
         pb.finish_and_clear();
-
-        let _ = t.stop();
+        rep.push(t.stop());
 
         fs::File::create(output.join(SENTINEL_INDEXED))?;
 
         let idx = KmerIndex::open(output)?;
+        let t_pack = Stage::start("pack");
         idx.pack_matrices()?;
+        rep.push(t_pack.stop());
         Ok(idx)
     }
 
@@ -98,6 +100,7 @@ impl KmerIndex {
         specs: &[OutputCol],
         threshold: u32,
         output_presence: bool,
+        rep: &mut Reporter,
     ) -> OKIResult<()> {
         if self.state() != IndexState::Indexed {
             return Err(OKIError::NotIndexed(self.root_path.clone()));
@@ -106,7 +109,6 @@ impl KmerIndex {
         let n_src_genomes = self.meta.genomes.len();
         let n_partitions  = self.partition.n_partitions();
 
-        // Open a second handle to the same path so we can borrow src and dst simultaneously.
         let src_partition = KmerPartition::open_with_config(
             &self.root_path,
             self.meta.config.kmer_size,
@@ -132,17 +134,17 @@ impl KmerIndex {
         ).map_err(OKIError::Partition)?;
 
         pb.finish_and_clear();
+        rep.push(t.stop());
 
-        let _ = t.stop();
-
-        // Update index.meta with new genome list and with_counts flag.
         self.meta.config.with_counts = !output_presence;
         self.meta.genomes = specs.iter()
             .map(|s| GenomeInfo::new(s.label.clone()))
             .collect();
         self.meta.write(&self.root_path)?;
 
+        let t_pack = Stage::start("pack");
         self.pack_matrices()?;
+        rep.push(t_pack.stop());
         Ok(())
     }
 }

src/obikmer/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "obikmer"
-version = "0.1.0"
+version = "1.1.38"
 edition = "2024"
 
 [[bin]]
@@ -18,7 +18,8 @@ obikrope      = { path = "../obikrope" }
 obikpartitionner = { path = "../obikpartitionner" }
 obisys        = { path = "../obisys" }
 obiskio       = { path = "../obiskio" }
-obikindex     = { path = "../obikindex" }
+obikindex     = { path = "../obikindex", default-features = false }
+obitaxonomy   = { path = "../obitaxonomy" }
 obilayeredmap = { path = "../obilayeredmap" }
 clap          = { version = "4", features = ["derive"] }
 serde_json    = "1"
@@ -32,4 +33,6 @@ tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
 pprof = { version = "0.13", features = ["prost-codec"], optional = true }
 
 [features]
+default   = ["numa"]
+numa      = ["obikindex/numa"]
 profiling = ["dep:pprof"]

src/obikmer/src/cmd/predicate.rs

@@ -3,6 +3,7 @@ use std::collections::HashMap;
 use clap::Args;
 use obikindex::GenomeInfo;
 use obikpartitionner::{GroupQuorumFilter, KmerFilter};
+use obitaxonomy::{TaxPath, TaxPattern};
 
 // ── Operator ──────────────────────────────────────────────────────────────────
 
@@ -49,7 +50,6 @@ impl MetaPred {
         if values.iter().any(|v| v.is_empty()) {
             return Err(format!("empty value in predicate: {s}"));
         }
-
         Ok(Self { key, op, values })
     }
 
@@ -70,18 +70,15 @@ impl MetaPred {
 
 // ── Path matching ─────────────────────────────────────────────────────────────
 
-/// True if `value` is equal to `pattern` or is a descendant of it in a `/`-separated hierarchy.
+/// True if the stored taxonomy `value` matches `pattern`.
 ///
-/// - Absolute pattern (`/a/b`): `value` must start with `/a/b` at a segment boundary.
-/// - Bare segment (`b`): `value` must contain `b` as an exact segment anywhere.
+/// `value` must be a valid `TaxPath` (starts with `taxonomy:/`).
+/// `pattern` is a `TaxPattern` query (see `obitaxonomy::TaxPattern` for syntax).
+/// Returns `false` if either fails to parse.
 fn path_matches(value: &str, pattern: &str) -> bool {
-    if pattern.starts_with('/') {
-        value == pattern
-            || (value.starts_with(pattern)
-                && value[pattern.len()..].starts_with('/'))
-    } else {
-        value.split('/').any(|seg| seg == pattern)
-    }
+    let Ok(path) = TaxPath::parse(value)    else { return false };
+    let Ok(pat)  = TaxPattern::parse(pattern) else { return false };
+    pat.matches(&path)
 }
 
 // ── Three-value group evaluation ──────────────────────────────────────────────

src/obikmer/src/cmd/query.rs

@@ -2,21 +2,26 @@ use std::collections::{HashMap, VecDeque};
 use std::io::{self, BufWriter, Write};
 use std::path::PathBuf;
 use std::sync::Arc;
+use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
+use std::time::Instant;
 
 use clap::Args;
 use obikindex::KmerIndex;
+use obikpartitionner::{KmerDesc, QueryHit, QueryStats};
 use obikrope::Rope;
-use obikseq::RoutableSuperKmer;
+use obikseq::CanonicalKmer;
 use obilayeredmap::IndexMode;
+use obipipeline::{Throttled, ThrottleGuard, throttle};
 use obiread::chunk::read_sequence_chunks_sized;
 use obiread::record::{SeqRecord, parse_chunk};
 use obiskbuilder::SuperKmerIter;
-use obisys::available_memory_bytes;
-use tracing::info;
+use obisys::{Reporter, Stage, available_memory_bytes, spinner};
+use tracing::{debug, info};
 
 // ── Pipeline data ─────────────────────────────────────────────────────────────
 
 enum QueryData {
+    Path(Throttled<PathBuf>),
     Chunk(Rope),
     Output(Vec<u8>),
 }
@@ -74,21 +79,34 @@ pub struct QueryArgs {
     /// I/O chunk size in MiB (default: auto-sized from available RAM and thread count)
     #[arg(long)]
     pub chunk_size: Option<usize>,
+
+    /// Maximum number of input files open simultaneously.
+    /// Defaults to threads/4 (minimum 1). Keep below the number of workers
+    /// to ensure CPU workers are always available for the transform stage.
+    #[arg(long)]
+    pub max_open_files: Option<usize>,
 }
 
-// ── SKDesc — one occurrence of a superkmer in the batch ───────────────────────
-
-/// Describes one occurrence of a superkmer in the query batch.
-pub struct SKDesc {
-    /// Index of the source sequence within the batch.
-    pub seq_idx: u32,
-    /// Kmer offset of the first kmer of this superkmer within its sequence.
-    pub kmer_offset: u32,
+impl QueryArgs {
+    pub fn effective_max_open(&self) -> usize {
+        self.max_open_files
+            .unwrap_or_else(|| (self.threads / 4).max(1))
+            .max(1)
+    }
 }
 
 // ── QueryBatch ────────────────────────────────────────────────────────────────
 
-/// A batch of query sequences with their superkmers deduplicated.
+/// A batch of query sequences, with k-mers deduplicated directly (not just at
+/// the superkmer level) and pre-split by partition.
+///
+/// Superkmer *construction* (`SuperKmerIter`) is still required — it's the
+/// mechanism that computes minimizers and partition routing — but the dedup
+/// key is the canonical k-mer, not the superkmer: two different superkmers
+/// that happen to share a k-mer (read overlaps, repeats, a SNP splitting an
+/// otherwise-identical run) are deduplicated too, not just identical whole
+/// superkmers. This also means each unique k-mer triggers at most one MPHF
+/// lookup, not one per occurrence.
 pub struct QueryBatch {
     /// Sequence ids in batch order.
     pub ids: Vec<String>,
@@ -96,30 +114,40 @@ pub struct QueryBatch {
     pub seqs: Vec<Vec<u8>>,
     /// Total kmer count per sequence (used for `--detail` coverage allocation).
     pub n_kmers: Vec<u32>,
-    /// Deduplicated superkmer map.
-    pub map: HashMap<RoutableSuperKmer, Vec<SKDesc>>,
+    /// Deduplicated k-mer occurrences, one map per partition.
+    pub by_partition: Vec<HashMap<CanonicalKmer, Vec<KmerDesc>>>,
 }
 
 impl QueryBatch {
-    /// Build a batch from a vec of parsed sequence records.
-    pub fn from_records(records: Vec<SeqRecord>, k: usize, level_max: usize, theta: f64) -> Self {
+    /// Build a batch from a vec of parsed sequence records, deduplicating
+    /// k-mers and routing them to partitions in the same pass.
+    pub fn from_records(
+        records: Vec<SeqRecord>,
+        k: usize,
+        level_max: usize,
+        theta: f64,
+        n_partitions: usize,
+    ) -> Self {
         let mut ids = Vec::with_capacity(records.len());
         let mut seqs = Vec::with_capacity(records.len());
         let mut n_kmers = Vec::with_capacity(records.len());
-        // Upper-bound estimate: at most one superkmer per k bases.
-        // Avoids repeated rehash on large chunks.
-        let cap = records.iter().map(|r| r.normalized.len()).sum::<usize>() / k.max(1);
-        let mut map: HashMap<RoutableSuperKmer, Vec<SKDesc>> = HashMap::with_capacity(cap);
+        let mask = (n_partitions as u64) - 1;
+        let mut by_partition: Vec<HashMap<CanonicalKmer, Vec<KmerDesc>>> =
+            (0..n_partitions).map(|_| HashMap::new()).collect();
 
         for (seq_idx, record) in records.into_iter().enumerate() {
             let mut kmer_offset = 0u32;
 
             for rsk in SuperKmerIter::new(&record.normalized, k, level_max, theta) {
+                let part_idx = (rsk.minimizer().seq_hash() & mask) as usize;
+                let map = &mut by_partition[part_idx];
+                for (j, kmer) in rsk.superkmer().iter_canonical_kmers().enumerate() {
+                    map.entry(kmer).or_default().push(KmerDesc {
+                        seq_idx: seq_idx as u32,
+                        pos: kmer_offset + j as u32,
+                    });
+                }
                 let n = (rsk.seql() - k + 1) as u32;
-                map.entry(rsk).or_default().push(SKDesc {
-                    seq_idx: seq_idx as u32,
-                    kmer_offset,
-                });
                 kmer_offset += n;
             }
 
@@ -132,37 +160,27 @@ impl QueryBatch {
             ids,
             seqs,
             n_kmers,
-            map,
+            by_partition,
         }
     }
-
-    /// Split the superkmer map by partition index.
-    pub fn split_by_partition(&self, n_partitions: usize) -> Vec<Vec<&RoutableSuperKmer>> {
-        let mask = (n_partitions as u64) - 1;
-        let mut by_part: Vec<Vec<&RoutableSuperKmer>> = vec![Vec::new(); n_partitions];
-        for rsk in self.map.keys() {
-            let part = (rsk.minimizer().seq_hash() & mask) as usize;
-            by_part[part].push(rsk);
-        }
-        by_part
-    }
 }
 
-// ── KmerResults — allocation-free ragged result matrix ───────────────────────
+// ── SmerIndex — sparse "was this k-mer found at all" bookkeeping ─────────────
 
-/// Flat storage for per-kmer query results across all sequences in a chunk.
-///
-/// Replaces `Vec<Vec<Option<Box<[u32]>>>>` — a single allocation for the whole
-/// chunk instead of one `Box<[u32]>` per found k-mer.
-struct KmerResults {
-    data: Vec<u32>,      // total_kmers × n_genomes, row-major
-    in_index: Vec<bool>, // total_kmers — true if the kmer was found in the index
-    offsets: Vec<usize>, // offsets[i]..offsets[i+1] = kmer range for sequence i
-    n_genomes: usize,
+/// Tracks, per (sequence, s-mer position), whether the k-mer was found in the
+/// index at all — independent of *which* genome(s) matched. Sized
+/// `total_smers` (one `bool` per s-mer occurrence in the chunk), **not**
+/// multiplied by `n_genomes`: this is the O(1)-per-position bookkeeping that
+/// `kmer_missing` needs (the leftmost-s-mer-of-window membership test), kept
+/// dense because it's already cheap — the `n_genomes`-scaled data lives in
+/// the sparse per-genome hit lists built alongside it (see `process_chunk`).
+struct SmerIndex {
+    in_index: Vec<bool>, // total_smers
+    offsets: Vec<usize>, // offsets[i]..offsets[i+1] = s-mer range for sequence i
 }
 
-impl KmerResults {
-    fn new(n_kmers_per_seq: &[u32], n_genomes: usize) -> Self {
+impl SmerIndex {
+    fn new(n_kmers_per_seq: &[u32]) -> Self {
         let mut offsets = Vec::with_capacity(n_kmers_per_seq.len() + 1);
         let mut total = 0usize;
         offsets.push(0);
@@ -171,34 +189,96 @@ impl KmerResults {
             offsets.push(total);
         }
         Self {
-            data: vec![0u32; total * n_genomes],
             in_index: vec![false; total],
             offsets,
-            n_genomes,
         }
     }
 
-    fn n_kmers_for(&self, seq: usize) -> usize {
-        self.offsets[seq + 1] - self.offsets[seq]
-    }
-
-    fn set(&mut self, seq: usize, kmer: usize, row: &[u32]) {
+    /// Mark the k-mer at (seq, kmer) as found in the index — independent of
+    /// any particular genome's value. Called once per hit k-mer (stage 1 of
+    /// `query_partition_with`), regardless of how the column-major fetch
+    /// (stage 2) later reports per-genome values.
+    fn mark_found(&mut self, seq: usize, kmer: usize) {
         let abs = self.offsets[seq] + kmer;
         self.in_index[abs] = true;
-        let base = abs * self.n_genomes;
-        self.data[base..base + self.n_genomes].copy_from_slice(row);
     }
 
     #[inline]
     fn is_in_index(&self, seq: usize, kmer: usize) -> bool {
         self.in_index[self.offsets[seq] + kmer]
     }
+}
 
-    /// Value for genome `g` at (seq, kmer); meaningful only when `is_in_index`.
-    #[inline]
-    fn val(&self, seq: usize, kmer: usize, g: usize) -> u32 {
-        self.data[(self.offsets[seq] + kmer) * self.n_genomes + g]
+// ── Sparse Findere: per-genome run detection + sliding-window minimum ────────
+
+/// One confirmed z-window: genome `g`'s window ending at k-mer `pos` (the
+/// *leftmost* s-mer of the window, i.e. the k_user-mer's output position) is
+/// fully present and nonzero, with window-minimum `value`.
+type ConfirmedHit = (u32, u32, u32); // (seq_idx, pos_out, value)
+
+/// Reduce one genome's raw sparse s-mer hits — `(seq_idx, pos_smer, raw_value)`,
+/// unsorted, exactly as delivered by `QueryHit::Value` — into confirmed
+/// z-windows, without ever visiting a position that had no hit at all.
+///
+/// A z-window is confirmed only when all z s-mers in it are present *and*
+/// nonzero for this genome (matching the dense sliding-window's semantics,
+/// where "not in index" or a zero value both contribute 0 to the window
+/// minimum) — which can only happen inside a maximal run of consecutive
+/// `pos_smer` values for the same sequence. `hits` is sorted in place by
+/// `(seq_idx, pos_smer)` to expose those runs; the monotone-deque
+/// window-minimum then runs per run, on run-relative indices, identical in
+/// spirit to the dense version's whole-sequence scan.
+///
+/// Returns the confirmed hits plus `(n_runs, total_run_len)` for logging —
+/// a low average run length relative to `z` means most hits fail to form a
+/// complete window.
+fn sparse_findere_for_genome(
+    hits: &mut [(u32, u32, u32)],
+    z: usize,
+    presence: bool,
+    threshold: u32,
+) -> (Vec<ConfirmedHit>, usize, usize) {
+    hits.sort_unstable_by_key(|&(seq, pos, _)| (seq, pos));
+
+    let mut confirmed = Vec::new();
+    let mut n_runs = 0usize;
+    let mut total_run_len = 0usize;
+    let mut dq: VecDeque<(usize, u32)> = VecDeque::new(); // (run-relative index, value)
+
+    let mut i = 0;
+    while i < hits.len() {
+        let seq = hits[i].0;
+        let mut j = i + 1;
+        while j < hits.len() && hits[j].0 == seq && hits[j].1 == hits[j - 1].1 + 1 {
+            j += 1;
+        }
+        let run = &hits[i..j];
+        n_runs += 1;
+        total_run_len += run.len();
+
+        dq.clear();
+        for (k, &(_, pos, val)) in run.iter().enumerate() {
+            while dq.back().map_or(false, |&(_, v)| v >= val) {
+                dq.pop_back();
+            }
+            dq.push_back((k, val));
+            while dq.front().map_or(false, |&(fk, _)| fk + z <= k) {
+                dq.pop_front();
+            }
+            if k + 1 >= z {
+                let win_min = dq.front().unwrap().1;
+                if win_min > 0 {
+                    let pos_out = pos + 1 - z as u32;
+                    let c = if presence { u32::from(win_min >= threshold) } else { win_min };
+                    confirmed.push((seq, pos_out, c));
+                }
+            }
+        }
+
+        i = j;
     }
+
+    (confirmed, n_runs, total_run_len)
 }
 
 // ── Per-sequence accumulator ──────────────────────────────────────────────────
@@ -234,40 +314,91 @@ fn process_chunk(
     force_presence: bool,
     presence_threshold: u32,
 ) -> Vec<u8> {
+    let chunk_start = Instant::now();
+    let chunk_bytes = rope.len();
+
     let records = parse_chunk(&rope, k);
     if records.is_empty() {
         return Vec::new();
     }
 
-    let batch = QueryBatch::from_records(records, k, 6, 0.7);
+    let batch = QueryBatch::from_records(records, k, 6, 0.7, n_partitions);
     let n_seqs = batch.ids.len();
 
-    // Flat result matrix — one allocation for the whole chunk.
-    let mut results = KmerResults::new(&batch.n_kmers, n_genomes);
+    // Estimate QueryBatch::by_partition's actual memory footprint: the
+    // k-mer-level dedup map (roadmap point 5) — one HashMap<CanonicalKmer,
+    // Vec<KmerDesc>> per partition, sized by *unique* k-mers, not shrunk by
+    // dedup. On real workloads with a low intra-chunk duplication rate this
+    // can dwarf every other per-chunk structure, including the sparse
+    // Findere ones logged further down — unlike those, chunk_bytes's formula
+    // (run()) does not account for this at all today. Measured by allocated
+    // capacity, not logical length, to reflect real memory pressure
+    // (HashMap/Vec growth slack) — `by_partition` is alive for the entire
+    // process_chunk call (never drained, only iterated by reference), so
+    // this is its footprint for the whole chunk lifetime, not a transient.
+    let hashmap_slot_bytes = (std::mem::size_of::<CanonicalKmer>()
+        + std::mem::size_of::<Vec<KmerDesc>>()
+        + 1) as u64; // +1 ≈ hashbrown control byte per slot
+    let by_partition_map_bytes: u64 = batch
+        .by_partition
+        .iter()
+        .map(|m| m.capacity() as u64 * hashmap_slot_bytes)
+        .sum();
+    let by_partition_desc_bytes: u64 = batch
+        .by_partition
+        .iter()
+        .flat_map(|m| m.values())
+        .map(|v| v.capacity() as u64 * std::mem::size_of::<KmerDesc>() as u64)
+        .sum();
+    let by_partition_bytes = by_partition_map_bytes + by_partition_desc_bytes;
 
-    let by_part = batch.split_by_partition(n_partitions);
+    debug!(
+        n_unique_kmers_total = batch.by_partition.iter().map(|m| m.len() as u64).sum::<u64>(),
+        by_partition_map_bytes,
+        by_partition_desc_bytes,
+        by_partition_bytes,
+        chunk_bytes,
+        "by_partition memory retained"
+    );
 
-    for (part_idx, part_sks) in by_part.iter().enumerate() {
-        if part_sks.is_empty() {
+    // Sparse bookkeeping for the whole chunk:
+    // - smer_index: O(total_smers) — is this s-mer in the index at all.
+    // - by_genome[g]: raw (seq_idx, pos_smer, value) hits for genome g, only
+    //   ever containing nonzero entries (query_partition_with never emits a
+    //   QueryHit::Value for a zero value) — empty for every genome this chunk
+    //   never matched, which is the common case for unrelated queries.
+    let mut smer_index = SmerIndex::new(&batch.n_kmers);
+    let mut by_genome: Vec<Vec<(u32, u32, u32)>> = (0..n_genomes).map(|_| Vec::new()).collect();
+
+    // Dedup-ratio bookkeeping: occurrences (from batch.n_kmers, computed
+    // before dedup) vs. unique k-mers actually queried (query_stats) — the
+    // entire justification for k-mer-level dereplication (see query.md,
+    // Future work point 5). If this ratio stays close to 1.0 on real data,
+    // dereplication isn't paying for itself and that should show up here.
+    let n_occurrences: u64 = batch.n_kmers.iter().map(|&n| n as u64).sum();
+    let mut query_stats = QueryStats::default();
+
+    for (part_idx, kmers) in batch.by_partition.iter().enumerate() {
+        if kmers.is_empty() {
             continue;
         }
 
-        idx.partition()
+        let stats = idx.partition()
             .query_partition_with(
                 part_idx,
-                part_sks,
-                k,
+                kmers,
                 n_genomes,
                 with_counts,
-                |sk_idx, kmer_idx, row| {
-                    let rsk = part_sks[sk_idx];
-                    let descs = batch.map.get(rsk).expect("rsk must be in map");
-                    for desc in descs {
-                        results.set(
-                            desc.seq_idx as usize,
-                            desc.kmer_offset as usize + kmer_idx,
-                            row,
-                        );
+                |event| match event {
+                    QueryHit::Found(descs) => {
+                        for desc in descs {
+                            smer_index.mark_found(desc.seq_idx as usize, desc.pos as usize);
+                        }
+                    }
+                    QueryHit::Value(descs, g, v) => {
+                        for desc in descs {
+                            by_genome[g].push((desc.seq_idx, desc.pos, v));
+                        }
                     }
                 },
             )
@@ -275,96 +406,129 @@ fn process_chunk(
                 eprintln!("query error on partition {part_idx}: {e}");
                 std::process::exit(1);
             });
+        query_stats += stats;
     }
 
-    // Sliding window minimum — one reusable buffer and one deque per batch.
-    //
-    // win_min[pos * n_genomes + g] = min count across the z-window [pos, pos+z)
-    // for genome g, where "not in index" counts as 0.
-    //
-    // win_min > 0  ↔  all z consecutive kmers are in the index with count > 0
-    //               ↔  Findere confirmation (for z=1 this degenerates to the
-    //                  simple case with no overhead).
-    //
-    // Works uniformly for count matrices and presence/absence (0/1) matrices.
-    let max_n_kmers = batch.n_kmers.iter().map(|&n| n as usize).max().unwrap_or(0);
-    let mut win_min = vec![0u32; max_n_kmers * n_genomes];
+    debug!(
+        n_occurrences,
+        n_unique_kmers = query_stats.n_unique_kmers,
+        n_mphf_calls = query_stats.n_mphf_calls,
+        n_hits = query_stats.n_hits,
+        n_columns_scanned = query_stats.n_columns_scanned,
+        n_col_get_calls = query_stats.n_col_get_calls,
+        "k-mer dedup + column-major fetch"
+    );
 
-    let mut accs: Vec<SeqAcc> = (0..n_seqs).map(|_| SeqAcc::new(n_genomes)).collect();
+    // ── Sparse Findere: per-genome run detection + sliding-window minimum ────
+    //
+    // Confirmed z-windows, per genome, replace the dense win_min matrix:
+    // total retained memory is O(actual hits), not O(total_smers × n_genomes)
+    // — the whole point of this pass. See sparse_findere_for_genome's doc for
+    // why run detection is equivalent to the dense scan's semantics.
+    let presence  = force_presence || !with_counts;
+    let threshold = presence_threshold;
+    let z         = effective_z;
 
     let n_kmers_out: Vec<usize> = batch
         .n_kmers
         .iter()
         .map(|&n| {
             let n = n as usize;
-            if n >= effective_z { n - effective_z + 1 } else { 0 }
+            if n >= z { n - z + 1 } else { 0 }
         })
         .collect();
+    let mut out_offsets = Vec::with_capacity(n_seqs + 1);
+    {
+        let mut total = 0usize;
+        out_offsets.push(0);
+        for &n in &n_kmers_out {
+            total += n;
+            out_offsets.push(total);
+        }
+    }
+    let total_out = *out_offsets.last().unwrap_or(&0);
 
+    let n_dense_would_be = n_occurrences as u64 * n_genomes as u64;
+    let mut n_sparse_entries = 0u64;
+    let mut n_runs_total = 0usize;
+    let mut run_len_total = 0usize;
+
+    let mut confirmed_by_genome: Vec<Vec<ConfirmedHit>> = Vec::with_capacity(n_genomes);
+    for hits in &mut by_genome {
+        n_sparse_entries += hits.len() as u64;
+        let (confirmed, n_runs, run_len) = sparse_findere_for_genome(hits, z, presence, threshold);
+        n_runs_total += n_runs;
+        run_len_total += run_len;
+        confirmed_by_genome.push(confirmed);
+    }
+
+    debug!(
+        n_dense_would_be,
+        n_sparse_entries,
+        n_runs = n_runs_total,
+        avg_run_len = if n_runs_total > 0 { run_len_total as f64 / n_runs_total as f64 } else { 0.0 },
+        z,
+        "sparse Findere"
+    );
+
+    // Actual bytes retained by the sparse hit structures (by_genome +
+    // confirmed_by_genome, both alive simultaneously at this point — see the
+    // chunk-size formula's comment in `run()`), by allocated capacity rather
+    // than logical length so this reflects real memory pressure including
+    // Vec growth slack. `empirical_multiplier` is directly comparable to
+    // BYTES_PER_KMER_PER_GENOME (`run()`) — the ratio a cluster run's logs
+    // need to judge whether that constant is over- or under-conservative for
+    // real data, instead of guessing.
+    const HIT_ENTRY_BYTES: u64 = std::mem::size_of::<(u32, u32, u32)>() as u64;
+    let by_genome_bytes: u64 = by_genome.iter().map(|v| v.capacity() as u64 * HIT_ENTRY_BYTES).sum();
+    let confirmed_bytes: u64 = confirmed_by_genome.iter().map(|v| v.capacity() as u64 * HIT_ENTRY_BYTES).sum();
+    let retained_bytes = by_genome_bytes + confirmed_bytes;
+
+    debug!(
+        by_genome_bytes,
+        confirmed_bytes,
+        retained_bytes,
+        chunk_bytes,
+        empirical_multiplier = retained_bytes as f64 / chunk_bytes.max(1) as f64,
+        "sparse memory retained"
+    );
+
+    // ── Accumulate: genome totals (per genome, from confirmed hits) ──────────
+    let mut accs: Vec<SeqAcc> = (0..n_seqs).map(|_| SeqAcc::new(n_genomes)).collect();
+    let mut confirmed_any = vec![false; total_out];
+
+    for (g, hits) in confirmed_by_genome.iter().enumerate() {
+        for &(seq_idx, pos_out, c) in hits {
+            let abs_out = out_offsets[seq_idx as usize] + pos_out as usize;
+            confirmed_any[abs_out] = true;
+            accs[seq_idx as usize].genome_totals[g] += c;
+        }
+    }
+
+    // ── Accumulate: kmer_count / kmer_missing (per position, genome-independent) ─
+    for seq_idx in 0..n_seqs {
+        let out_n = n_kmers_out[seq_idx];
+        let acc = &mut accs[seq_idx];
+        for pos in 0..out_n {
+            let abs_out = out_offsets[seq_idx] + pos;
+            if confirmed_any[abs_out] {
+                acc.kmer_count += 1;
+            } else if !smer_index.is_in_index(seq_idx, pos) {
+                acc.kmer_missing += 1;
+            }
+        }
+    }
+
+    // ── Coverage (--detail): densify only when actually requested ────────────
     let mut cov: Vec<Vec<Vec<u32>>> = if detail {
         n_kmers_out.iter().map(|&n| vec![vec![0u32; n]; n_genomes]).collect()
     } else {
         Vec::new()
     };
-
-    let presence  = force_presence || !with_counts;
-    let threshold = presence_threshold;
-    let z         = effective_z;
-
-    // Deque reused across all (seq, genome) pairs.
-    let mut dq: VecDeque<(usize, u32)> = VecDeque::with_capacity(z + 1);
-
-    for seq_idx in 0..n_seqs {
-        let n     = results.n_kmers_for(seq_idx);
-        let out_n = n_kmers_out[seq_idx];
-        if out_n == 0 { continue; }
-
-        let mins = &mut win_min[..out_n * n_genomes];
-        mins.fill(0);
-
-        // ── Per-genome sliding window minimum ─────────────────────────────────
-        for g in 0..n_genomes {
-            dq.clear();
-            for i in 0..n {
-                let v_i = if results.is_in_index(seq_idx, i) {
-                    results.val(seq_idx, i, g)
-                } else {
-                    0
-                };
-                // Evict elements that have left the window.
-                while dq.front().map_or(false, |&(f, _)| f + z <= i) {
-                    dq.pop_front();
-                }
-                // Maintain monotone non-decreasing back→front for minimum at front.
-                while dq.back().map_or(false, |&(_, v)| v >= v_i) {
-                    dq.pop_back();
-                }
-                dq.push_back((i, v_i));
-                // Window [pos, pos+z) is complete when i = pos + z - 1.
-                if i + 1 >= z {
-                    let pos = i + 1 - z;
-                    mins[pos * n_genomes + g] = dq.front().unwrap().1;
-                }
-            }
-        }
-
-        // ── Accumulate ────────────────────────────────────────────────────────
-        let acc = &mut accs[seq_idx];
-        for pos in 0..out_n {
-            let any = (0..n_genomes).any(|g| mins[pos * n_genomes + g] > 0);
-            if !any {
-                if !results.is_in_index(seq_idx, pos) {
-                    acc.kmer_missing += 1;
-                }
-                continue;
-            }
-            acc.kmer_count += 1;
-            for g in 0..n_genomes {
-                let v = mins[pos * n_genomes + g];
-                if v == 0 { continue; }
-                let c = if presence { u32::from(v >= threshold) } else { v };
-                acc.genome_totals[g] += c;
-                if detail { cov[seq_idx][g][pos] += c; }
+    if detail {
+        for (g, hits) in confirmed_by_genome.iter().enumerate() {
+            for &(seq_idx, pos_out, c) in hits {
+                cov[seq_idx as usize][g][pos_out as usize] += c;
             }
         }
     }
@@ -384,9 +548,42 @@ fn process_chunk(
         &cov,
         &mut buf,
     );
+
+    debug!(
+        chunk_bytes,
+        n_seqs,
+        n_smers = batch.n_kmers.iter().map(|&n| n as u64).sum::<u64>(),
+        wall_ms = chunk_start.elapsed().as_millis() as u64,
+        "process_chunk"
+    );
+
     buf
 }
 
+// ── GuardedChunkIter — keeps the throttle slot guard alive until the file is exhausted ──
+
+/// Wraps a per-file `Rope` chunk iterator together with its `ThrottleGuard`,
+/// so the guard (and the throttle slot it holds) is only released once the
+/// file has been fully read — never earlier, never held past that point.
+struct GuardedChunkIter {
+    inner:      Box<dyn Iterator<Item = Rope> + Send>,
+    _guard:     ThrottleGuard,
+    files_open: Arc<AtomicU32>,
+}
+
+impl Iterator for GuardedChunkIter {
+    type Item = Rope;
+    fn next(&mut self) -> Option<Rope> {
+        self.inner.next()
+    }
+}
+
+impl Drop for GuardedChunkIter {
+    fn drop(&mut self) {
+        self.files_open.fetch_sub(1, Ordering::Relaxed);
+    }
+}
+
 // ── Entry point ───────────────────────────────────────────────────────────────
 
 pub fn run(args: QueryArgs) {
@@ -402,17 +599,67 @@ pub fn run(args: QueryArgs) {
     let n_workers = args.threads.max(1);
 
     // Chunk size: each chunk stays in memory for its entire processing lifetime.
-    // Overhead per raw byte is ~8× (Rope + parsed records + superkmers + results).
-    // We target ≤ 50 % of available RAM across all concurrent workers.
+    //
+    // Per-chunk memory is no longer a dense n_genomes-wide buffer (removed in
+    // the sparse Findere rework, see process_chunk) — it now scales with
+    // *actual hit count*, not with total_kmers_in_chunk × n_genomes
+    // unconditionally. BYTES_PER_KMER_PER_GENOME below is therefore a
+    // pathological-case bound, not a typical-case estimate: it protects
+    // against a fully-dense hit pattern (every k-mer of the query matching
+    // every genome — a degenerate case, e.g. low-complexity input theta-
+    // filtering should mostly reject, or an index of near-duplicate genomes),
+    // where by_genome and confirmed_by_genome (process_chunk) both end up
+    // holding one (seq_idx, pos, value) entry — 3 × u32 = 12 bytes, vs. 4
+    // bytes for the old dense encoding, where position was implicit in the
+    // array index — per (k-mer, genome) pair, and *coexist simultaneously*
+    // (by_genome isn't freed before confirmed_by_genome is built), for a
+    // worst case of ~24 bytes/pair before Vec growth slack. `cov` remains
+    // fully dense when --detail is set (unaffected by the sparse rework),
+    // still roughly doubling the n_genomes-scaled cost.
+    //
+    // For realistic, sparse hit patterns actual memory is far below this
+    // bound — see the "sparse memory retained" debug log in process_chunk,
+    // which reports the empirical bytes-per-raw-byte multiplier actually
+    // observed per chunk, directly comparable to BYTES_PER_KMER_PER_GENOME
+    // below. Tightening this constant for typical-case throughput (at the
+    // cost of pathological-case safety margin) is a deliberate tuning
+    // decision to make from that data, not something to guess at here.
+    //
+    // BASE_OVERHEAD approximates what scales with chunk_bytes alone,
+    // independent of n_genomes: the Rope itself, parsed SeqRecord sequence +
+    // normalised bytes, the superkmer dedup map, and the JSON output buffer.
+    // Like the n_genomes-scaled term, this is an estimate — validate against
+    // actual peak RSS (Stage::stop's `rss` in the summary table) on real
+    // workloads rather than trusting it blindly.
+    //
+    // We target ≤ 50 % of available RAM across all concurrent workers
+    // (SAFETY_FACTOR).
+    const BASE_OVERHEAD: u64 = 4;
+    const BYTES_PER_KMER_PER_GENOME: u64 = 8; // pathological-case bound — see comment above
+    const SAFETY_FACTOR: u64 = 2;
+
+    let detail_factor: u64 = if args.detail { 2 } else { 1 };
+    let overhead_multiplier =
+        BASE_OVERHEAD + n_genomes as u64 * BYTES_PER_KMER_PER_GENOME * detail_factor;
+
     let chunk_bytes = args
         .chunk_size
         .map(|mb| mb * 1024 * 1024)
         .unwrap_or_else(|| {
             let avail = available_memory_bytes();
-            let computed = avail / (n_workers as u64 * 16);
+            let computed = avail / (n_workers as u64 * overhead_multiplier * SAFETY_FACTOR);
             computed.clamp(4 * 1024 * 1024, 256 * 1024 * 1024) as usize
         });
 
+    debug!(
+        chunk_bytes,
+        n_genomes,
+        detail = args.detail,
+        overhead_multiplier,
+        estimated_peak_chunk_bytes = chunk_bytes as u64 * overhead_multiplier,
+        "chunk-size formula resolved"
+    );
+
     let effective_z: usize = args
         .findere_z
         .unwrap_or_else(|| match idx.meta().config.evidence {
@@ -435,48 +682,122 @@ pub fn run(args: QueryArgs) {
     let force_presence = args.force_presence;
     let presence_threshold = args.presence_threshold;
 
-    // Flat iterator over all Rope chunks from all input files.
-    // I/O runs in the source thread; chunk processing is parallelised by the pipe.
-    info!("query: chunk_size={}MiB", chunk_bytes / (1024 * 1024));
+    // Throttled iterator over input file paths: at most `effective_max_open()`
+    // files are open at once. Opening + decompressing + chunking each file is
+    // now a Flat pipeline stage, executed across the `n_workers` pool — not
+    // serialised in the pipe's dedicated source thread (see steps::scatter /
+    // cmd::superkmer for the same pattern applied to indexing).
+    info!("query: chunk_size={}MiB, max_open_files={}", chunk_bytes / (1024 * 1024), args.effective_max_open());
 
     let paths: Vec<PathBuf> = args.inputs.iter().map(PathBuf::from).collect();
-    let all_chunks = paths.into_iter().flat_map(move |path| {
-        let path_str = path.to_str().unwrap_or("").to_owned();
-        match read_sequence_chunks_sized(&path_str, chunk_bytes) {
-            Ok(iter) => Box::new(iter.filter_map(|r| match r {
-                Ok(rope) => Some(rope),
-                Err(e) => {
-                    eprintln!("read error: {e}");
-                    None
-                }
-            })) as Box<dyn Iterator<Item = Rope> + Send>,
-            Err(e) => {
-                eprintln!("error opening {path_str}: {e}");
-                std::process::exit(1);
-            }
-        }
-    });
+    let path_source = throttle(paths.into_iter(), args.effective_max_open());
+
+    // Instrumentation: total bytes processed (for the EMA throughput readout),
+    // number of files currently open/being chunked, and number of chunks
+    // currently being processed by a worker — all read from the spinner loop
+    // below, updated from inside the pipe closures.
+    let total_bytes = Arc::new(AtomicU64::new(0));
+    let files_open = Arc::new(AtomicU32::new(0));
+    let chunks_active = Arc::new(AtomicU32::new(0));
 
     let pipe = obipipeline::make_pipe! {
-        QueryData : Rope => Vec<u8>,
+        QueryData : Throttled<PathBuf> => Vec<u8>,
+        || {
+            let files_open = Arc::clone(&files_open);
+            move |pw: Throttled<PathBuf>| -> GuardedChunkIter {
+                let path = pw.item;
+                let guard = pw.guard;
+                let path_str = path.to_str().unwrap_or("").to_owned();
+                files_open.fetch_add(1, Ordering::Relaxed);
+                let open_start = Instant::now();
+                // Hard-exit on file-open failure (mirrors the previous behaviour):
+                // propagating this as a pipeline Err would hit a known scheduler
+                // hang on early stage errors (obipipeline::scheduler::WorkerPool::run
+                // breaks its main loop without unblocking the still-running source
+                // thread, so the final `h.join()` never returns) — worth fixing in
+                // obipipeline itself, but out of scope here; sidestepping it like the
+                // original code already did is the safe choice for this change.
+                let iter = read_sequence_chunks_sized(&path_str, chunk_bytes).unwrap_or_else(|e| {
+                    eprintln!("error opening {path_str}: {e}");
+                    std::process::exit(1);
+                });
+                debug!(
+                    path = %path_str,
+                    open_ms = open_start.elapsed().as_millis() as u64,
+                    "opened query input file"
+                );
+                let err_path = path_str.clone();
+                GuardedChunkIter {
+                    inner: Box::new(iter.filter_map(move |r| match r {
+                        Ok(rope) => Some(rope),
+                        Err(e) => {
+                            eprintln!("read error: {err_path}: {e}");
+                            None
+                        }
+                    })),
+                    _guard: guard,
+                    files_open: Arc::clone(&files_open),
+                }
+            }
+        } : Path => Chunk,
         | {
             let idx = Arc::clone(&idx);
+            let total_bytes = Arc::clone(&total_bytes);
+            let chunks_active = Arc::clone(&chunks_active);
             move |rope: Rope| {
-                process_chunk(
+                chunks_active.fetch_add(1, Ordering::Relaxed);
+                let bytes = rope.len() as u64;
+                let out = process_chunk(
                     &idx, rope, k, n_genomes, n_partitions, with_counts,
                     effective_z, detail, count_missing, force_presence, presence_threshold,
-                )
+                );
+                total_bytes.fetch_add(bytes, Ordering::Relaxed);
+                chunks_active.fetch_sub(1, Ordering::Relaxed);
+                out
             }
         } : Chunk => Output,
     };
 
+    let t = Stage::start("query");
+    let pb = spinner("query");
+
+    let mut ema_rate: f64 = 0.0;
+    let mut last_t = Instant::now();
+    let mut last_bytes: u64 = 0;
+    const ALPHA: f64 = 0.15;
+
     let mut out = BufWriter::new(io::stdout());
-    for block in pipe.apply(all_chunks, n_workers, 2) {
+    for block in pipe.apply(path_source, n_workers, 2) {
         if !block.is_empty() {
             out.write_all(&block).expect("write error");
         }
+
+        let now = Instant::now();
+        let dt = now.duration_since(last_t).as_secs_f64();
+        if dt > 0.1 {
+            let total = total_bytes.load(Ordering::Relaxed);
+            let instant = (total - last_bytes) as f64 / dt;
+            ema_rate = ALPHA * instant + (1.0 - ALPHA) * ema_rate;
+            last_t = now;
+            last_bytes = total;
+            let bp = total as f64;
+            let (count_str, rate_str) = if bp >= 1e9 {
+                (format!("{:.2} GB", bp / 1e9), format!("{:.0} MB/s", ema_rate / 1e6))
+            } else {
+                (format!("{:.0} MB", bp / 1e6), format!("{:.0} MB/s", ema_rate / 1e6))
+            };
+            let active = chunks_active.load(Ordering::Relaxed);
+            let open = files_open.load(Ordering::Relaxed);
+            pb.set_message(format!("{count_str}  {rate_str}  [files open: {open}, chunks in flight: {active}]"));
+        }
     }
     out.flush().expect("flush error");
+
+    pb.finish_and_clear();
+
+    let mut rep = Reporter::new();
+    rep.push(t.stop());
+    rep.print();
 }
 
 // ── Output ────────────────────────────────────────────────────────────────────
@@ -501,7 +822,9 @@ fn emit_batch(
 
         let mut match_map = serde_json::Map::new();
         for (g, genome) in meta.genomes.iter().enumerate() {
-            match_map.insert(genome.label.clone(), acc.genome_totals[g].into());
+            if acc.genome_totals[g] != 0 {
+                match_map.insert(genome.label.clone(), acc.genome_totals[g].into());
+            }
         }
         ann.insert("kmer_strict_matches".into(), match_map.into());
 
@@ -524,3 +847,7 @@ fn emit_batch(
         let _ = out.write_all(b"\n");
     }
 }
+
+#[cfg(test)]
+#[path = "tests/query.rs"]
+mod tests;

src/obikmer/src/cmd/select.rs

@@ -4,6 +4,7 @@ use std::path::PathBuf;
 use clap::{Args, ValueEnum};
 use obikindex::{GenomeInfo, KmerIndex};
 use obikpartitionner::{AggOp, OutputCol};
+use obisys::Reporter;
 use tracing::info;
 
 use super::predicate::matching_genome_indices;
@@ -229,20 +230,24 @@ pub fn run(args: SelectArgs) {
         if output_presence { "presence" } else { "count" },
     );
 
+    let mut rep = Reporter::new();
+
     if args.in_place {
-        src.select_in_place(&specs, args.presence_threshold, output_presence)
+        src.select_in_place(&specs, args.presence_threshold, output_presence, &mut rep)
             .unwrap_or_else(|e| {
                 eprintln!("select error: {e}");
                 std::process::exit(1);
             });
+        rep.print();
         info!("selected in-place → {}", args.source.display());
     } else {
         let output = args.output.unwrap();
-        KmerIndex::select(&output, &src, &specs, args.presence_threshold, output_presence, args.force)
+        KmerIndex::select(&output, &src, &specs, args.presence_threshold, output_presence, args.force, &mut rep)
             .unwrap_or_else(|e| {
                 eprintln!("select error: {e}");
                 std::process::exit(1);
             });
+        rep.print();
         info!("selected index → {}", output.display());
     }
 }

src/obikmer/src/cmd/tests/query.rs

@@ -0,0 +1,228 @@
+use super::*;
+
+const K: usize = 11;
+const M: usize = 5;
+
+/// Build a `QueryBatch` from raw FASTA text, going through the same
+/// `Rope` + `parse_chunk` path `process_chunk` uses — avoids hand-building a
+/// `normalized` `Rope`, which is an implementation detail of `obiread`.
+///
+/// `obikseq`'s global K/M params are thread-local under `test-utils` (see
+/// `obikseq::params`), so setting them here is per-test-thread and does not
+/// need coordination with other tests.
+fn batch_from_fasta(fasta: &str, k: usize, n_partitions: usize) -> QueryBatch {
+    obikseq::set_k(k);
+    obikseq::set_m(M);
+    let mut rope = Rope::new(Some("text/fasta"));
+    rope.push(fasta.as_bytes().to_vec());
+    let records = parse_chunk(&rope, k);
+    QueryBatch::from_records(records, k, 6, 0.7, n_partitions)
+}
+
+fn total_occurrences(batch: &QueryBatch) -> u64 {
+    batch.n_kmers.iter().map(|&n| n as u64).sum()
+}
+
+fn total_unique_kmers(batch: &QueryBatch) -> u64 {
+    batch.by_partition.iter().map(|m| m.len() as u64).sum()
+}
+
+// A 60 bp sequence, arbitrary but fixed — no attempt is made to prove it is
+// free of internal k=11 repeats; the tests below only rely on inequalities
+// that hold regardless (see each test's comment).
+const SEQ: &str = "CATTAGCGTACCTGATCAGGTTACAGCTTAGGCATCCAGTTGACCATGACTGGACTTAGC";
+
+#[test]
+fn single_sequence_yields_plausible_kmer_counts() {
+    // A single record can still contain internal repeats (SEQ isn't
+    // guaranteed repeat-free at k=11) — this only checks the batch is
+    // internally consistent, not a specific dedup ratio. The cross-record
+    // tests below make the actual, unconditional dedup claims.
+    let fasta = format!(">r1\n{SEQ}\n");
+    let batch = batch_from_fasta(&fasta, K, 1);
+
+    assert_eq!(batch.ids, vec!["r1".to_string()]);
+    let occurrences = total_occurrences(&batch);
+    let unique = total_unique_kmers(&batch);
+    assert!(occurrences > 0, "sequence should yield at least one k-mer");
+    assert!(unique > 0 && unique <= occurrences);
+}
+
+#[test]
+fn duplicated_sequence_across_records_deduplicates() {
+    // Two records with byte-identical sequences: every k-mer in record 1
+    // exactly duplicates one in record 0, so unique kmers <= n_kmers[0],
+    // strictly less than the summed occurrences (2 * n_kmers[0]) as long as
+    // the sequence yields at least one k-mer. This holds regardless of
+    // whether SEQ has internal repeats.
+    let fasta = format!(">r1\n{SEQ}\n>r2\n{SEQ}\n");
+    let batch = batch_from_fasta(&fasta, K, 1);
+
+    assert_eq!(batch.ids.len(), 2);
+    let occurrences = total_occurrences(&batch);
+    let unique = total_unique_kmers(&batch);
+
+    assert!(batch.n_kmers[0] > 0);
+    assert_eq!(occurrences, batch.n_kmers[0] as u64 + batch.n_kmers[1] as u64);
+    assert!(
+        unique <= batch.n_kmers[0] as u64,
+        "identical sequences must not produce more unique k-mers than one copy has"
+    );
+    assert!(
+        unique < occurrences,
+        "k-mer-level dedup must collapse at least the cross-record duplication"
+    );
+}
+
+#[test]
+fn duplicated_sequence_broadcasts_to_both_seq_indices() {
+    // Stronger than the ratio check above: pick any k-mer that hit in both
+    // records and confirm its occurrence list actually references both
+    // seq_idx 0 and seq_idx 1 — this is the specific new capability (dedup
+    // reaching across records/superkmers), not just a smaller unique count.
+    let fasta = format!(">r1\n{SEQ}\n>r2\n{SEQ}\n");
+    let batch = batch_from_fasta(&fasta, K, 1);
+
+    let shared = batch.by_partition[0]
+        .values()
+        .find(|descs| descs.iter().any(|d| d.seq_idx == 0) && descs.iter().any(|d| d.seq_idx == 1));
+
+    assert!(
+        shared.is_some(),
+        "expected at least one k-mer shared between the two identical records"
+    );
+}
+
+#[test]
+fn empty_records_yield_empty_batch() {
+    let batch = batch_from_fasta("", K, 1);
+    assert!(batch.ids.is_empty());
+    assert_eq!(total_occurrences(&batch), 0);
+    assert_eq!(total_unique_kmers(&batch), 0);
+}
+
+#[test]
+fn partition_routing_is_a_pure_function_of_the_kmer() {
+    // With n_partitions=4, every occurrence of a given k-mer must land in
+    // the same partition bucket as every other occurrence of that k-mer
+    // (partition routing is derived from the minimizer, shared by
+    // definition among instances of the same k-mer's containing superkmer
+    // in this test's single-sequence-pair setup).
+    let fasta = format!(">r1\n{SEQ}\n>r2\n{SEQ}\n");
+    let batch = batch_from_fasta(&fasta, K, 4);
+
+    let total_unique: u64 = batch.by_partition.iter().map(|m| m.len() as u64).sum();
+    assert!(total_unique > 0);
+
+    // No k-mer key appears in more than one partition's map.
+    let mut seen: std::collections::HashSet<CanonicalKmer> = std::collections::HashSet::new();
+    for map in &batch.by_partition {
+        for kmer in map.keys() {
+            assert!(seen.insert(*kmer), "k-mer routed to more than one partition");
+        }
+    }
+}
+
+// ── sparse_findere_for_genome vs. a dense reference implementation ──────────
+//
+// No property-testing crate (proptest/quickcheck) is a workspace dependency
+// (checked before writing this — not adding one for a single test module,
+// per this project's dependency-approval rule). A tiny deterministic xorshift
+// PRNG, std-only, stands in for one.
+
+/// Faithful reimplementation of the pre-phase-5 dense sliding-window scan —
+/// the algorithm `sparse_findere_for_genome` replaced — used here only as a
+/// correctness oracle, not in production code. Operates on one genome's
+/// hits across possibly many sequences, exactly like the sparse version.
+fn dense_reference_findere(
+    hits: &[(u32, u32, u32)],
+    seq_lens: &[usize],
+    z: usize,
+    presence: bool,
+    threshold: u32,
+) -> Vec<(u32, u32, u32)> {
+    let mut by_seq: Vec<Vec<u32>> = seq_lens.iter().map(|&n| vec![0u32; n]).collect();
+    for &(seq, pos, val) in hits {
+        by_seq[seq as usize][pos as usize] = val;
+    }
+
+    let mut confirmed = Vec::new();
+    for (seq_idx, values) in by_seq.iter().enumerate() {
+        let n = values.len();
+        let mut dq: std::collections::VecDeque<(usize, u32)> = std::collections::VecDeque::new();
+        for i in 0..n {
+            let v_i = values[i];
+            while dq.front().map_or(false, |&(f, _)| f + z <= i) {
+                dq.pop_front();
+            }
+            while dq.back().map_or(false, |&(_, v)| v >= v_i) {
+                dq.pop_back();
+            }
+            dq.push_back((i, v_i));
+            if i + 1 >= z {
+                let win_min = dq.front().unwrap().1;
+                if win_min > 0 {
+                    let pos_out = (i + 1 - z) as u32;
+                    let c = if presence { u32::from(win_min >= threshold) } else { win_min };
+                    confirmed.push((seq_idx as u32, pos_out, c));
+                }
+            }
+        }
+    }
+    confirmed
+}
+
+/// Minimal std-only xorshift64 PRNG — deterministic, seedable, no dependency.
+struct Xorshift64(u64);
+impl Xorshift64 {
+    fn next(&mut self) -> u64 {
+        self.0 ^= self.0 << 13;
+        self.0 ^= self.0 >> 7;
+        self.0 ^= self.0 << 17;
+        self.0
+    }
+    fn range(&mut self, n: u32) -> u32 {
+        (self.next() % n as u64) as u32
+    }
+}
+
+#[test]
+fn sparse_findere_matches_dense_reference_on_random_inputs() {
+    let mut rng = Xorshift64(0x5eed_5eed_5eed_5eedu64);
+
+    for case in 0..200 {
+        let n_seqs = 1 + rng.range(4) as usize;
+        let seq_lens: Vec<usize> = (0..n_seqs).map(|_| 1 + rng.range(30) as usize).collect();
+        let z = 1 + rng.range(4) as usize;
+        let presence = rng.range(2) == 0;
+        let threshold = 1 + rng.range(3);
+
+        // Sparse density varies across cases, including edge cases (empty,
+        // fully dense) — deliberately not uniform, to stress both few-hits
+        // and many-overlapping-runs scenarios.
+        let density = rng.range(101);
+        let mut hits: Vec<(u32, u32, u32)> = Vec::new();
+        for (seq_idx, &len) in seq_lens.iter().enumerate() {
+            for pos in 0..len {
+                if rng.range(100) < density {
+                    let val = 1 + rng.range(5); // never 0 — matches QueryHit::Value's invariant
+                    hits.push((seq_idx as u32, pos as u32, val));
+                }
+            }
+        }
+
+        let mut sparse_input = hits.clone();
+        let (mut sparse_result, _, _) =
+            sparse_findere_for_genome(&mut sparse_input, z, presence, threshold);
+        let mut dense_result = dense_reference_findere(&hits, &seq_lens, z, presence, threshold);
+
+        sparse_result.sort_unstable();
+        dense_result.sort_unstable();
+
+        assert_eq!(
+            sparse_result, dense_result,
+            "case {case}: n_seqs={n_seqs} seq_lens={seq_lens:?} z={z} presence={presence} \
+             threshold={threshold} density={density} hits={hits:?}"
+        );
+    }
+}

src/obikmer/src/main.rs

@@ -6,7 +6,7 @@ use clap::{Parser, Subcommand};
 use tracing_subscriber::{EnvFilter, fmt};
 
 #[derive(Parser)]
-#[command(name = "obikmer", about = "DNA k-mer tools")]
+#[command(name = "obikmer", about = "DNA k-mer tools", version)]
 struct Cli {
     #[command(subcommand)]
     command: Commands,

src/obikpartitionner/src/common.rs

@@ -3,7 +3,6 @@ use std::io;
 use std::path::{Path, PathBuf};
 
 use obicompactvec::{PersistentBitVecBuilder, PersistentCompactIntVecBuilder};
-use obicompactvec::traits::BitSliceMut;
 use obilayeredmap::meta::PartitionMeta;
 use obilayeredmap::{IndexMode, OLMError};
 use obiskio::{SKError, SKResult};

src/obikpartitionner/src/filter.rs

@@ -1,9 +1,24 @@
+use obicompactvec::FilterMask;
+
 /// Trait for kmer row filters.
 ///
 /// `row` contains raw per-genome counts (or 0/1 for presence/absence data).
 /// `n_genomes` equals `row.len()`.
 pub trait KmerFilter: Send + Sync {
     fn passes(&self, row: &[u32], n_genomes: usize) -> bool;
+
+    /// Express this filter as a [`FilterMask`] column-operation expression.
+    ///
+    /// Returns `Some(expr)` if the filter can be evaluated solely from matrix
+    /// column aggregates (no per-kmer row scan needed).  Returns `None` if the
+    /// filter requires row-level inspection.
+    ///
+    /// `threshold` semantics in the returned mask use `>= threshold`, matching
+    /// [`obicompactvec::MatrixGroupOps`].  Implementations must add 1 to any
+    /// row-level threshold that uses strict `>` comparison.
+    fn column_mask_expr(&self, _n_genomes: usize) -> Option<FilterMask> {
+        None
+    }
 }
 
 /// True when `row` passes every filter in `filters`.
@@ -29,6 +44,16 @@ impl KmerFilter for MinGenomeFraction {
         let p = present_count(row, self.threshold);
         p as f64 / n_genomes as f64 >= self.frac
     }
+
+    fn column_mask_expr(&self, n_genomes: usize) -> Option<FilterMask> {
+        let t = self.threshold.checked_add(1)?;
+        let min_count = (self.frac * n_genomes as f64).ceil() as usize;
+        Some(FilterMask::PresenceGeq {
+            indices: (0..n_genomes).collect(),
+            threshold: t,
+            min_count,
+        })
+    }
 }
 
 /// At most `frac` fraction of genomes contain this kmer (count > `threshold`).
@@ -42,6 +67,16 @@ impl KmerFilter for MaxGenomeFraction {
         let p = present_count(row, self.threshold);
         p as f64 / n_genomes as f64 <= self.frac
     }
+
+    fn column_mask_expr(&self, n_genomes: usize) -> Option<FilterMask> {
+        let t = self.threshold.checked_add(1)?;
+        let max_count = (self.frac * n_genomes as f64).floor() as usize;
+        Some(FilterMask::PresenceLeq {
+            indices: (0..n_genomes).collect(),
+            threshold: t,
+            max_count,
+        })
+    }
 }
 
 /// At least `count` genomes contain this kmer (count > `threshold`).
@@ -54,6 +89,15 @@ impl KmerFilter for MinGenomeCount {
     fn passes(&self, row: &[u32], _n_genomes: usize) -> bool {
         present_count(row, self.threshold) >= self.count
     }
+
+    fn column_mask_expr(&self, n_genomes: usize) -> Option<FilterMask> {
+        let t = self.threshold.checked_add(1)?;
+        Some(FilterMask::PresenceGeq {
+            indices: (0..n_genomes).collect(),
+            threshold: t,
+            min_count: self.count,
+        })
+    }
 }
 
 /// At most `count` genomes contain this kmer (count > `threshold`).
@@ -66,6 +110,15 @@ impl KmerFilter for MaxGenomeCount {
     fn passes(&self, row: &[u32], _n_genomes: usize) -> bool {
         present_count(row, self.threshold) <= self.count
     }
+
+    fn column_mask_expr(&self, n_genomes: usize) -> Option<FilterMask> {
+        let t = self.threshold.checked_add(1)?;
+        Some(FilterMask::PresenceLeq {
+            indices: (0..n_genomes).collect(),
+            threshold: t,
+            max_count: self.count,
+        })
+    }
 }
 
 // ── Total-count filters (count indexes only) ───────────────────────────────────
@@ -79,6 +132,13 @@ impl KmerFilter for MinTotalCount {
     fn passes(&self, row: &[u32], _n_genomes: usize) -> bool {
         row.iter().sum::<u32>() >= self.total
     }
+
+    fn column_mask_expr(&self, n_genomes: usize) -> Option<FilterMask> {
+        Some(FilterMask::SumGeq {
+            indices: (0..n_genomes).collect(),
+            min_sum: self.total,
+        })
+    }
 }
 
 /// Sum of counts across all genomes <= `total`.
@@ -90,6 +150,13 @@ impl KmerFilter for MaxTotalCount {
     fn passes(&self, row: &[u32], _n_genomes: usize) -> bool {
         row.iter().sum::<u32>() <= self.total
     }
+
+    fn column_mask_expr(&self, n_genomes: usize) -> Option<FilterMask> {
+        Some(FilterMask::SumLeq {
+            indices: (0..n_genomes).collect(),
+            max_sum: self.total,
+        })
+    }
 }
 
 // ── Group-based quorum filter ─────────────────────────────────────────────────
@@ -113,6 +180,37 @@ pub struct GroupQuorumFilter {
     pub max_outgroup_frac:  f64,
 }
 
+impl GroupQuorumFilter {
+    // Build PresenceGeq/PresenceLeq constraints for one group (ingroup or outgroup).
+    fn group_mask_parts(
+        indices: &[usize],
+        threshold: u32,
+        min_count: usize,
+        max_count: usize,
+        min_frac: f64,
+        max_frac: f64,
+        parts: &mut Vec<FilterMask>,
+    ) {
+        let n = indices.len();
+        let geq = min_count.max((min_frac * n as f64).ceil() as usize);
+        if geq > 0 {
+            parts.push(FilterMask::PresenceGeq {
+                indices: indices.to_vec(),
+                threshold,
+                min_count: geq,
+            });
+        }
+        let leq = max_count.min((max_frac * n as f64).floor() as usize);
+        if leq < n {
+            parts.push(FilterMask::PresenceLeq {
+                indices: indices.to_vec(),
+                threshold,
+                max_count: leq,
+            });
+        }
+    }
+}
+
 impl KmerFilter for GroupQuorumFilter {
     fn passes(&self, row: &[u32], _n_genomes: usize) -> bool {
         if !self.ingroup_idx.is_empty() {
@@ -139,4 +237,26 @@ impl KmerFilter for GroupQuorumFilter {
         }
         true
     }
+
+    fn column_mask_expr(&self, _n_genomes: usize) -> Option<FilterMask> {
+        let t = self.threshold.checked_add(1)?;
+        let mut parts: Vec<FilterMask> = Vec::new();
+        if !self.ingroup_idx.is_empty() {
+            Self::group_mask_parts(
+                &self.ingroup_idx, t,
+                self.min_count, self.max_count,
+                self.min_frac, self.max_frac,
+                &mut parts,
+            );
+        }
+        if !self.outgroup_idx.is_empty() {
+            Self::group_mask_parts(
+                &self.outgroup_idx, t,
+                self.min_outgroup_count, self.max_outgroup_count,
+                self.min_outgroup_frac, self.max_outgroup_frac,
+                &mut parts,
+            );
+        }
+        Some(FilterMask::And(parts))
+    }
 }

src/obikpartitionner/src/lib.rs

@@ -14,4 +14,5 @@ mod select_layer;
 pub use filter::{GroupQuorumFilter, KmerFilter, passes_all};
 pub use merge_layer::MergeMode;
 pub use partition::{KmerPartition, KmerSpectrum, PARTITIONS_SUBDIR};
+pub use query_layer::{KmerDesc, QueryHit, QueryStats};
 pub use select_layer::{AggOp, OutputCol};

src/obikpartitionner/src/merge_layer.rs

@@ -10,6 +10,7 @@ use obipipeline::{
 };
 
 use obicompactvec::{
+    MatrixGroupOps,
     PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentBitVecBuilder,
     PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentCompactIntVecBuilder,
 };
@@ -78,6 +79,41 @@ impl SrcLayerData {
         }
         buf
     }
+
+    pub(crate) fn n_slots(&self) -> usize {
+        match self {
+            SrcLayerData::Presence(_, mat) => mat.n(),
+            SrcLayerData::Count(_, mat) => mat.n(),
+        }
+    }
+
+    /// MPHF lookup: returns the slot index for `kmer` (kmer must be in the domain).
+    #[inline]
+    pub(crate) fn slot(&self, kmer: CanonicalKmer) -> usize {
+        match self {
+            SrcLayerData::Presence(mphf, _) => mphf.index(kmer),
+            SrcLayerData::Count(mphf, _) => mphf.index(kmer),
+        }
+    }
+
+    /// Row lookup by slot index, bypassing the MPHF.
+    #[inline]
+    pub(crate) fn fill_row_by_slot(&self, slot: usize, n_genomes: usize) -> Vec<u32> {
+        let mut buf = vec![0u32; n_genomes];
+        match self {
+            SrcLayerData::Presence(_, mat) => mat.fill_row(slot, &mut buf),
+            SrcLayerData::Count(_, mat) => mat.fill_row(slot, &mut buf),
+        }
+        buf
+    }
+
+    /// Call `f` with a reference to the underlying matrix as `&dyn MatrixGroupOps`.
+    pub(crate) fn with_matrix<R>(&self, f: impl FnOnce(&dyn MatrixGroupOps) -> R) -> R {
+        match self {
+            SrcLayerData::Presence(_, mat) => f(mat),
+            SrcLayerData::Count(_, mat) => f(mat),
+        }
+    }
 }
 
 // ── helpers ───────────────────────────────────────────────────────────────────

src/obikpartitionner/src/query_layer.rs

@@ -1,7 +1,8 @@
+use std::collections::HashMap;
 use std::path::Path;
 
 use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
-use obikseq::{CanonicalKmer, RoutableSuperKmer};
+use obikseq::CanonicalKmer;
 use obiskio::{SKError, SKResult};
 use obilayeredmap::{IndexMode, MphfLayer, OLMError};
 use obilayeredmap::meta::PartitionMeta;
@@ -44,53 +45,133 @@ impl QueryLayer {
         }
     }
 
-    /// Write per-genome values into `buf` if `kmer` is indexed; returns true on hit.
-    fn find_into(&self, kmer: CanonicalKmer, n_genomes: usize, buf: &mut [u32]) -> bool {
+    /// MPHF lookup only — no matrix access. `Some(slot)` on hit.
+    fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
         match self {
-            QueryLayer::Presence(mphf, mat) => {
-                if let Some(slot) = mphf.find(kmer) {
-                    mat.fill_row(slot, &mut buf[..n_genomes]);
-                    true
-                } else {
-                    false
-                }
-            }
-            QueryLayer::Count(mphf, mat) => {
-                if let Some(slot) = mphf.find(kmer) {
-                    mat.fill_row(slot, &mut buf[..n_genomes]);
-                    true
-                } else {
-                    false
-                }
-            }
+            QueryLayer::Presence(mphf, _) | QueryLayer::Count(mphf, _) => mphf.find(kmer),
+        }
+    }
+
+    /// Number of genome columns this layer's matrix actually has. Bounds
+    /// column-major iteration — usually equal to the index's `n_genomes`, but
+    /// `PersistentBitMatrix::Implicit` (the documented mono-genome fast path)
+    /// always reports exactly `1`, regardless of the index's real genome
+    /// count, so callers must use this rather than assuming `n_genomes`.
+    fn n_cols(&self) -> usize {
+        match self {
+            QueryLayer::Presence(_, mat) => mat.n_cols(),
+            QueryLayer::Count(_, mat) => mat.n_cols(),
+        }
+    }
+
+    /// Column-major point lookup: value for genome column `g` at `slot`.
+    /// `g` must be `< self.n_cols()`; `slot` must come from [`find_slot`] on
+    /// this same layer.
+    fn col_value(&self, g: usize, slot: usize) -> u32 {
+        match self {
+            QueryLayer::Presence(_, mat) => mat.get(g, slot),
+            QueryLayer::Count(_, mat) => mat.col_view(g).get(slot),
         }
     }
 }
 
-// ── KmerPartition::query_partition* ──────────────────────────────────────────
+// ── KmerDesc — one occurrence of a k-mer in the query batch ──────────────────
+
+/// Describes one occurrence of a (deduplicated) k-mer in the query batch:
+/// which sequence it came from, and its absolute s-mer position within it.
+#[derive(Debug, Clone, Copy)]
+pub struct KmerDesc {
+    pub seq_idx: u32,
+    pub pos: u32,
+}
+
+/// Aggregate counters for one `query_partition_with` call — feeds the
+/// dedup-ratio and column-scan logging in `obikmer::cmd::query` (occurrences
+/// vs. unique k-mers is the whole justification for k-mer-level
+/// dereplication; columns scanned / `get()` calls quantify the column-major
+/// fetch's locality claim).
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub struct QueryStats {
+    /// Distinct canonical k-mers queried in this partition.
+    pub n_unique_kmers: usize,
+    /// Total `MphfLayer::find` calls issued (a k-mer tried against more than
+    /// one layer before a hit, or against all layers on a miss, counts once
+    /// per layer attempted).
+    pub n_mphf_calls: usize,
+    /// Distinct canonical k-mers that matched some layer.
+    pub n_hits: usize,
+    /// Total genome columns scanned across all hit layers (sum of
+    /// `layer.n_cols()` over layers with at least one hit).
+    pub n_columns_scanned: usize,
+    /// Total `col_value` calls issued during the column-major fetch pass
+    /// (`n_columns_scanned` × hits-per-layer, summed over layers).
+    pub n_col_get_calls: usize,
+}
+
+impl std::ops::AddAssign for QueryStats {
+    fn add_assign(&mut self, other: Self) {
+        self.n_unique_kmers += other.n_unique_kmers;
+        self.n_mphf_calls += other.n_mphf_calls;
+        self.n_hits += other.n_hits;
+        self.n_columns_scanned += other.n_columns_scanned;
+        self.n_col_get_calls += other.n_col_get_calls;
+    }
+}
+
+// ── QueryHit — one event delivered to query_partition_with's callback ───────
+
+/// One event from [`KmerPartition::query_partition_with`]'s two-stage query:
+/// a `Found` event once per hit k-mer (stage 1, MPHF-only — mark the k-mer as
+/// indexed regardless of any genome's value), then a `Value` event per
+/// `(hit k-mer, genome)` pair with a nonzero matrix value (stage 2,
+/// column-major fetch). Carried as one enum, not two separate callbacks, so
+/// the caller only needs one `FnMut` closure — passing two closures that each
+/// need to mutably borrow the same accumulator does not borrow-check.
+pub enum QueryHit<'a> {
+    Found(&'a [KmerDesc]),
+    Value(&'a [KmerDesc], usize, u32),
+}
+
+// ── KmerPartition::query_partition_with ──────────────────────────────────────
 
 impl KmerPartition {
-    /// Query a single partition, calling `on_hit(sk_idx, kmer_idx, row)` for
-    /// every found k-mer without allocating intermediate result vectors.
+    /// Query a single partition for a pre-deduplicated map of canonical
+    /// k-mers → their occurrences (`seq_idx`, `pos`) in the query batch.
+    ///
+    /// Two stages:
+    /// 1. **MPHF-only pass**: for each unique k-mer, try each layer's MPHF in
+    ///    turn (stopping at the first hit) and bucket confirmed hits by
+    ///    `(layer, slot)`. Emits one `QueryHit::Found` per hit k-mer. This
+    ///    stage's cost is independent of the index's genome count.
+    /// 2. **Column-major fetch**: for each layer with at least one hit, walk
+    ///    its matrix **column by column** (genome by genome) — for each
+    ///    genome, scan the slots bucketed in stage 1 and look up their value.
+    ///    Emits one `QueryHit::Value` per nonzero `(k-mer, genome)` pair.
+    ///    Total lookups are the same as a row-major pass (`n_hits × n_cols`
+    ///    in the worst case); the win is memory locality — both persistent
+    ///    matrix formats are column-oriented on disk (one `mmap`'d region per
+    ///    genome), so scanning one column at a time touches far fewer
+    ///    distinct mmap regions than fetching one full row per hit.
     pub fn query_partition_with<F>(
         &self,
         part_idx: usize,
-        superkmers: &[&RoutableSuperKmer],
-        _k: usize,
+        kmers: &HashMap<CanonicalKmer, Vec<KmerDesc>>,
         n_genomes: usize,
         with_counts: bool,
-        mut on_hit: F,
-    ) -> SKResult<()>
+        mut on_event: F,
+    ) -> SKResult<QueryStats>
     where
-        F: FnMut(usize, usize, &[u32]),
+        F: FnMut(QueryHit),
     {
-        if superkmers.is_empty() {
-            return Ok(());
+        let mut stats = QueryStats::default();
+
+        if kmers.is_empty() {
+            return Ok(stats);
         }
 
         let index_dir = self.part_dir(part_idx).join(INDEX_SUBDIR);
         if !index_dir.exists() {
-            return Ok(());
+            return Ok(stats);
         }
 
         let meta = PartitionMeta::load(&index_dir).map_err(olm_to_sk)?;
@@ -98,69 +179,47 @@ impl KmerPartition {
             .map(|i| QueryLayer::open(&index_dir.join(format!("layer_{i}")), with_counts, &meta.mode))
             .collect::<SKResult<_>>()?;
 
-        let mut buf = vec![0u32; n_genomes];
+        // ── Stage 1: MPHF-only pass, bucket hits by (layer_idx, slot) ────────
+        let mut by_layer: Vec<HashMap<usize, &Vec<KmerDesc>>> =
+            (0..layers.len()).map(|_| HashMap::new()).collect();
 
-        for (sk_idx, rsk) in superkmers.iter().enumerate() {
-            for (kmer_idx, kmer) in rsk.superkmer().iter_canonical_kmers().enumerate() {
-                for layer in &layers {
-                    if layer.find_into(kmer, n_genomes, &mut buf) {
-                        on_hit(sk_idx, kmer_idx, &buf);
-                        buf.fill(0);
-                        break;
+        for (kmer, descs) in kmers {
+            stats.n_unique_kmers += 1;
+            for (layer_idx, layer) in layers.iter().enumerate() {
+                stats.n_mphf_calls += 1;
+                if let Some(slot) = layer.find_slot(*kmer) {
+                    by_layer[layer_idx].insert(slot, descs);
+                    on_event(QueryHit::Found(descs));
+                    stats.n_hits += 1;
+                    break;
+                }
+            }
+        }
+
+        // ── Stage 2: column-major fetch, per layer ───────────────────────────
+        for (layer_idx, slots) in by_layer.iter().enumerate() {
+            if slots.is_empty() {
+                continue;
+            }
+            let layer = &layers[layer_idx];
+            let n_cols = layer.n_cols().min(n_genomes);
+            stats.n_columns_scanned += n_cols;
+
+            for g in 0..n_cols {
+                for (&slot, descs) in slots {
+                    stats.n_col_get_calls += 1;
+                    let v = layer.col_value(g, slot);
+                    if v != 0 {
+                        on_event(QueryHit::Value(descs, g, v));
                     }
                 }
             }
         }
 
-        Ok(())
-    }
-
-    /// Query a single partition for a slice of super-kmers, returning per-kmer rows.
-    /// Prefer [`query_partition_with`] to avoid per-kmer heap allocations.
-    pub fn query_partition(
-        &self,
-        part_idx: usize,
-        superkmers: &[&RoutableSuperKmer],
-        _k: usize,
-        n_genomes: usize,
-        with_counts: bool,
-    ) -> SKResult<Vec<Vec<Option<Box<[u32]>>>>> {
-        if superkmers.is_empty() {
-            return Ok(Vec::new());
-        }
-
-        let index_dir = self.part_dir(part_idx).join(INDEX_SUBDIR);
-
-        if !index_dir.exists() {
-            return Ok(superkmers
-                .iter()
-                .map(|rsk| vec![None; rsk.seql()])
-                .collect());
-        }
-
-        let meta = PartitionMeta::load(&index_dir).map_err(olm_to_sk)?;
-        let layers: Vec<QueryLayer> = (0..meta.n_layers)
-            .map(|i| QueryLayer::open(&index_dir.join(format!("layer_{i}")), with_counts, &meta.mode))
-            .collect::<SKResult<_>>()?;
-
-        let mut buf = vec![0u32; n_genomes];
-        Ok(superkmers
-            .iter()
-            .map(|rsk| {
-                rsk.superkmer()
-                    .iter_canonical_kmers()
-                    .map(|kmer| {
-                        for layer in &layers {
-                            if layer.find_into(kmer, n_genomes, &mut buf) {
-                                let row: Box<[u32]> = buf[..n_genomes].into();
-                                buf.fill(0);
-                                return Some(row);
-                            }
-                        }
-                        None
-                    })
-                    .collect()
-            })
-            .collect())
+        Ok(stats)
     }
 }
+
+#[cfg(test)]
+#[path = "tests/query_layer.rs"]
+mod tests;

src/obikpartitionner/src/rebuild_layer.rs

@@ -1,8 +1,9 @@
 use std::path::Path;
 
 use obicompactvec::{
-    PersistentBitMatrixBuilder, PersistentBitVecBuilder, PersistentCompactIntMatrixBuilder,
-    PersistentCompactIntVecBuilder,
+    FilterMask, eval_filter_mask,
+    PersistentBitMatrixBuilder, PersistentBitVecBuilder,
+    PersistentCompactIntMatrixBuilder, PersistentCompactIntVecBuilder,
 };
 use obidebruinj::GraphDeBruijn;
 use obikseq::CanonicalKmer;
@@ -10,18 +11,135 @@ use obilayeredmap::meta::PartitionMeta;
 use obilayeredmap::{IndexMode, MphfLayer};
 use obiskio::{SKError, SKResult, UnitigFileReader};
 
-use crate::common::{ColBuilder, col_path_bit, col_path_int, load_meta, olm_to_sk, write_matrix_meta};
-use crate::filter::{KmerFilter, passes_all};
+use crate::common::{load_meta, olm_to_sk};
+use crate::filter::KmerFilter;
 use crate::graph_pipeline::materialize_layer;
 use crate::merge_layer::{MergeMode, SrcLayerData};
 use crate::partition::KmerPartition;
 
 const INDEX_SUBDIR: &str = "index";
 
-/// Iterate all kmers in `src_index_dir` that pass `filters`, yielding `(kmer, row)`.
+// ── Builders — pair matrix builder + column builders for one mode ─────────────
+
+enum Builders {
+    Presence(PersistentBitMatrixBuilder, Vec<PersistentBitVecBuilder>),
+    Count(PersistentCompactIntMatrixBuilder, Vec<PersistentCompactIntVecBuilder>),
+}
+
+impl Builders {
+    fn new(mode: MergeMode, n: usize, dir: &Path, n_genomes: usize) -> SKResult<Self> {
+        match mode {
+            MergeMode::Presence => {
+                let mut mat = PersistentBitMatrixBuilder::new(n, dir).map_err(SKError::Io)?;
+                let mut cols = Vec::with_capacity(n_genomes);
+                for _ in 0..n_genomes { cols.push(mat.add_col().map_err(SKError::Io)?); }
+                Ok(Builders::Presence(mat, cols))
+            }
+            MergeMode::Count => {
+                let mut mat = PersistentCompactIntMatrixBuilder::new(n, dir).map_err(SKError::Io)?;
+                let mut cols = Vec::with_capacity(n_genomes);
+                for _ in 0..n_genomes { cols.push(mat.add_col().map_err(SKError::Io)?); }
+                Ok(Builders::Count(mat, cols))
+            }
+        }
+    }
+
+    fn set_val(&mut self, col: usize, slot: usize, value: u32) {
+        match self {
+            Builders::Presence(_, cols) => cols[col].set(slot, value > 0),
+            Builders::Count(_, cols)    => cols[col].set(slot, value),
+        }
+    }
+
+    fn close(self) -> SKResult<()> {
+        match self {
+            Builders::Presence(mat, cols) => {
+                for b in cols { b.close().map_err(SKError::Io)?; }
+                mat.close().map_err(SKError::Io)
+            }
+            Builders::Count(mat, cols) => {
+                for b in cols { b.close().map_err(SKError::Io)?; }
+                mat.close().map_err(SKError::Io)
+            }
+        }
+    }
+}
+
+// ── try_compute_combined_mask ─────────────────────────────────────────────────
+
+/// Build a per-slot `TempBitVec` mask from `filters` using column operations
+/// on the source matrix — no per-kmer MPHF lookup or row read needed.
 ///
-/// Uses [`SrcLayerData`] semantics: counts take priority over presence when
-/// `mode = Count`; presence (or implicit all-ones) is used for `Presence`.
+/// Returns `Some(mask)` when every filter in `filters` can express itself as
+/// a [`FilterMask`] expression.  Returns `None` when any filter requires
+/// row-level inspection (fall back to `passes_all`).
+fn try_compute_combined_mask(
+    filters: &[Box<dyn KmerFilter>],
+    src_data: &SrcLayerData,
+    n_genomes: usize,
+) -> SKResult<Option<obicompactvec::TempBitVec>> {
+    if filters.is_empty() {
+        return Ok(None);
+    }
+    let mut exprs: Vec<FilterMask> = Vec::with_capacity(filters.len());
+    for f in filters {
+        match f.column_mask_expr(n_genomes) {
+            Some(expr) => exprs.push(expr),
+            None => return Ok(None),
+        }
+    }
+    let combined = FilterMask::And(exprs);
+    let n = src_data.n_slots();
+    let mask = src_data
+        .with_matrix(|mat| eval_filter_mask(&combined, mat, n))
+        .map_err(SKError::Io)?;
+    Ok(Some(mask))
+}
+
+// ── iter_src_kmers_masked (pass 1) ────────────────────────────────────────────
+
+/// Iterate all passing kmers in `src_index_dir`, yielding only the kmer value.
+///
+/// When all filters can be expressed as column operations, a per-slot mask is
+/// computed once per layer and used for O(1) slot-check per kmer instead of a
+/// full row read.  Falls back to row-level `passes_all` otherwise.
+fn iter_src_kmers_masked(
+    src_index_dir: &Path,
+    mode: MergeMode,
+    n_genomes: usize,
+    filters: &[Box<dyn KmerFilter>],
+    mut cb: impl FnMut(CanonicalKmer),
+) -> SKResult<()> {
+    let src_meta = load_meta(src_index_dir, "rebuild")?;
+    for l in 0..src_meta.n_layers {
+        let src_layer_dir = src_index_dir.join(format!("layer_{l}"));
+        let unitigs_path = src_layer_dir.join("unitigs.bin");
+        if !unitigs_path.exists() { continue; }
+
+        let src_data = SrcLayerData::open(&src_layer_dir, mode)?;
+        let mask = try_compute_combined_mask(filters, &src_data, n_genomes)?;
+        let reader = UnitigFileReader::open_sequential(&unitigs_path)?;
+
+        for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
+            let slot = src_data.slot(kmer);
+            let passes = match &mask {
+                Some(m) => m.get(slot),
+                None => {
+                    let row = src_data.fill_row_by_slot(slot, n_genomes);
+                    filters.iter().all(|f| f.passes(&row, n_genomes))
+                }
+            };
+            if passes { cb(kmer); }
+        }
+    }
+    Ok(())
+}
+
+// ── iter_src_layers (pass 2) ──────────────────────────────────────────────────
+
+/// Iterate all passing kmers in `src_index_dir`, yielding `(kmer, row)`.
+///
+/// When the slot mask is available, skips the row read for filtered-out slots.
 fn iter_src_layers(
     src_index_dir: &Path,
     mode: MergeMode,
@@ -33,17 +151,23 @@ fn iter_src_layers(
     for l in 0..src_meta.n_layers {
         let src_layer_dir = src_index_dir.join(format!("layer_{l}"));
         let unitigs_path = src_layer_dir.join("unitigs.bin");
-        if !unitigs_path.exists() {
-            continue;
-        }
+        if !unitigs_path.exists() { continue; }
 
-        let reader = UnitigFileReader::open_sequential(&unitigs_path)?;
         let src_data = SrcLayerData::open(&src_layer_dir, mode)?;
+        let mask = try_compute_combined_mask(filters, &src_data, n_genomes)?;
+        let reader = UnitigFileReader::open_sequential(&unitigs_path)?;
 
         for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
-            let row = src_data.lookup(kmer, n_genomes);
-            if passes_all(filters, &row, n_genomes) {
+            let slot = src_data.slot(kmer);
+            if let Some(ref m) = mask {
+                if !m.get(slot) { continue; }
+                let row = src_data.fill_row_by_slot(slot, n_genomes);
                 cb(kmer, row.into_boxed_slice());
+            } else {
+                let row = src_data.fill_row_by_slot(slot, n_genomes);
+                if filters.iter().all(|f| f.passes(&row, n_genomes)) {
+                    cb(kmer, row.into_boxed_slice());
+                }
             }
         }
     }
@@ -81,7 +205,7 @@ impl KmerPartition {
 
         // ── Pass 1: collect filtered kmers into de Bruijn graph ───────────────
         let mut g = GraphDeBruijn::new();
-        iter_src_layers(&src_index_dir, mode, n_genomes, filters, |kmer, _row| {
+        iter_src_kmers_masked(&src_index_dir, mode, n_genomes, filters, |kmer| {
             g.push(kmer);
         })?;
 
@@ -100,54 +224,22 @@ impl KmerPartition {
         // ── Prepare matrix builders (one column per genome) ───────────────────
         let data_dir = match mode {
             MergeMode::Presence => dst_layer_dir.join("presence"),
-            MergeMode::Count => dst_layer_dir.join("counts"),
+            MergeMode::Count    => dst_layer_dir.join("counts"),
         };
         std::fs::create_dir_all(&data_dir)?;
-
-        let mut builders: Vec<ColBuilder> = match mode {
-            MergeMode::Presence => {
-                PersistentBitMatrixBuilder::new(n_new, &data_dir)
-                    .map_err(SKError::Io)?
-                    .close()
-                    .map_err(SKError::Io)?;
-                (0..n_genomes)
-                    .map(|g| -> SKResult<ColBuilder> {
-                        let b = PersistentBitVecBuilder::new(n_new, &col_path_bit(&data_dir, g))?;
-                        Ok(ColBuilder::Bit(b))
-                    })
-                    .collect::<SKResult<_>>()?
-            }
-            MergeMode::Count => {
-                PersistentCompactIntMatrixBuilder::new(n_new, &data_dir)
-                    .map_err(SKError::Io)?
-                    .close()
-                    .map_err(SKError::Io)?;
-                (0..n_genomes)
-                    .map(|g| -> SKResult<ColBuilder> {
-                        let b = PersistentCompactIntVecBuilder::new(
-                            n_new,
-                            &col_path_int(&data_dir, g),
-                        )?;
-                        Ok(ColBuilder::Int(b))
-                    })
-                    .collect::<SKResult<_>>()?
-            }
-        };
+        let mut builders = Builders::new(mode, n_new, &data_dir, n_genomes)?;
 
         // ── Pass 2: fill builders ─────────────────────────────────────────────
         iter_src_layers(&src_index_dir, mode, n_genomes, filters, |kmer, row| {
             if let Some(slot) = dst_mphf.find(kmer) {
                 for (col, &value) in row.iter().enumerate() {
-                    builders[col].set_val(slot, value);
+                    builders.set_val(col, slot, value);
                 }
             }
         })?;
 
-        // ── Close builders, write metadata ────────────────────────────────────
-        for b in builders {
-            b.close()?;
-        }
-        write_matrix_meta(&data_dir, n_new, n_genomes).map_err(SKError::Io)?;
+        // ── Close builders and write metadata ─────────────────────────────────
+        builders.close()?;
 
         PartitionMeta {
             n_layers: 1,

src/obikpartitionner/src/select_layer.rs

@@ -3,10 +3,10 @@ use std::io;
 use std::path::{Path, PathBuf};
 
 use obicompactvec::{
-    PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentBitVecBuilder,
-    PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentCompactIntVecBuilder,
+    ColGroup, MatrixGroupOps,
+    PersistentBitMatrix, PersistentBitMatrixBuilder,
+    PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
 };
-use obicompactvec::traits::BitSliceMut;
 use obilayeredmap::meta::PartitionMeta;
 use obilayeredmap::OLMError;
 use obiskio::{SKError, SKResult};
@@ -41,52 +41,6 @@ pub struct OutputCol {
     pub op:      AggOp,
 }
 
-// ── Aggregation ───────────────────────────────────────────────────────────────
-
-#[inline]
-fn aggregate(op: AggOp, indices: &[usize], src_row: &[u32], threshold: u32) -> u32 {
-    match op {
-        AggOp::Any => {
-            if indices.iter().any(|&i| src_row[i] > threshold) { 1 } else { 0 }
-        }
-        AggOp::All => {
-            if indices.is_empty() { return 0; }
-            if indices.iter().all(|&i| src_row[i] > threshold) { 1 } else { 0 }
-        }
-        AggOp::None => {
-            if indices.iter().all(|&i| src_row[i] <= threshold) { 1 } else { 0 }
-        }
-        AggOp::Sum => {
-            indices.iter().map(|&i| src_row[i]).fold(0u32, |a, b| a.saturating_add(b))
-        }
-        AggOp::Min => indices.iter().map(|&i| src_row[i]).min().unwrap_or(0),
-        AggOp::Max => indices.iter().map(|&i| src_row[i]).max().unwrap_or(0),
-    }
-}
-
-// ── ColBuilder ────────────────────────────────────────────────────────────────
-
-enum ColBuilder {
-    Bit(PersistentBitVecBuilder),
-    Int(PersistentCompactIntVecBuilder),
-}
-
-impl ColBuilder {
-    fn set_val(&mut self, slot: usize, value: u32) {
-        match self {
-            ColBuilder::Bit(b) => b.set(slot, value > 0),
-            ColBuilder::Int(b) => b.set(slot, value),
-        }
-    }
-
-    fn close(self) -> SKResult<()> {
-        match self {
-            ColBuilder::Bit(b) => b.close().map_err(SKError::Io),
-            ColBuilder::Int(b) => b.close().map_err(SKError::Io),
-        }
-    }
-}
-
 // ── Helpers ───────────────────────────────────────────────────────────────────
 
 fn olm_to_sk(e: OLMError) -> SKError {
@@ -96,21 +50,6 @@ fn olm_to_sk(e: OLMError) -> SKError {
     }
 }
 
-fn col_path_bit(dir: &Path, col: usize) -> PathBuf {
-    dir.join(format!("col_{col:06}.pbiv"))
-}
-
-fn col_path_int(dir: &Path, col: usize) -> PathBuf {
-    dir.join(format!("col_{col:06}.pciv"))
-}
-
-fn write_matrix_meta(dir: &Path, n: usize, n_cols: usize) -> io::Result<()> {
-    fs::write(
-        dir.join("meta.json"),
-        format!("{{\"n\":{n},\"n_cols\":{n_cols}}}\n"),
-    )
-}
-
 /// Copy all plain files (not subdirectories) from `src_dir` to `dst_dir`.
 fn copy_layer_files(src_dir: &Path, dst_dir: &Path) -> io::Result<()> {
     for entry in fs::read_dir(src_dir)? {
@@ -126,30 +65,64 @@ fn copy_layer_files(src_dir: &Path, dst_dir: &Path) -> io::Result<()> {
 // ── fill_builders ─────────────────────────────────────────────────────────────
 
 fn fill_builders(
-    builders: &mut [ColBuilder],
     specs: &[OutputCol],
-    n: usize,
-    n_src: usize,
     src_layer_dir: &Path,
     src_is_count: bool,
     threshold: u32,
+    output_presence: bool,
+    mut dst_bit: Option<&mut PersistentBitMatrixBuilder>,
+    mut dst_int: Option<&mut PersistentCompactIntMatrixBuilder>,
 ) -> SKResult<()> {
-    let mut src_buf = vec![0u32; n_src];
-
     if src_is_count {
         let mat = PersistentCompactIntMatrix::open(src_layer_dir).map_err(SKError::Io)?;
-        for slot in 0..n {
-            mat.fill_row(slot, &mut src_buf);
-            for (col, spec) in specs.iter().enumerate() {
-                builders[col].set_val(slot, aggregate(spec.op, &spec.indices, &src_buf, threshold));
+        for spec in specs {
+            let g = ColGroup::new(&spec.label, spec.indices.clone());
+            if output_presence {
+                let b = dst_bit.as_deref_mut().unwrap();
+                match spec.op {
+                    AggOp::Any  => b.add_col_from    (&mat.partial_group_any (&g, threshold).map_err(SKError::Io)?),
+                    AggOp::All  => b.add_col_from    (&mat.partial_group_all (&g, threshold).map_err(SKError::Io)?),
+                    AggOp::None => b.add_col_from    (&mat.partial_group_none(&g, threshold).map_err(SKError::Io)?),
+                    AggOp::Sum  => b.add_col_from_int(&mat.partial_group_sum (&g).map_err(SKError::Io)?),
+                    AggOp::Min  => b.add_col_from_int(&mat.partial_group_min (&g).map_err(SKError::Io)?),
+                    AggOp::Max  => b.add_col_from_int(&mat.partial_group_max (&g).map_err(SKError::Io)?),
+                }.map_err(SKError::Io)?;
+            } else {
+                let b = dst_int.as_deref_mut().unwrap();
+                match spec.op {
+                    AggOp::Sum  => b.add_col_from    (&mat.partial_group_sum (&g).map_err(SKError::Io)?),
+                    AggOp::Min  => b.add_col_from    (&mat.partial_group_min (&g).map_err(SKError::Io)?),
+                    AggOp::Max  => b.add_col_from    (&mat.partial_group_max (&g).map_err(SKError::Io)?),
+                    AggOp::Any  => b.add_col_from_bit(&mat.partial_group_any (&g, threshold).map_err(SKError::Io)?),
+                    AggOp::All  => b.add_col_from_bit(&mat.partial_group_all (&g, threshold).map_err(SKError::Io)?),
+                    AggOp::None => b.add_col_from_bit(&mat.partial_group_none(&g, threshold).map_err(SKError::Io)?),
+                }.map_err(SKError::Io)?;
             }
         }
     } else {
         let mat = PersistentBitMatrix::open(src_layer_dir).map_err(SKError::Io)?;
-        for slot in 0..n {
-            mat.fill_row(slot, &mut src_buf);
-            for (col, spec) in specs.iter().enumerate() {
-                builders[col].set_val(slot, aggregate(spec.op, &spec.indices, &src_buf, threshold));
+        for spec in specs {
+            let g = ColGroup::new(&spec.label, spec.indices.clone());
+            if output_presence {
+                let b = dst_bit.as_deref_mut().unwrap();
+                match spec.op {
+                    AggOp::Any  => b.add_col_from    (&mat.partial_group_any (&g, 1).map_err(SKError::Io)?),
+                    AggOp::All  => b.add_col_from    (&mat.partial_group_all (&g, 1).map_err(SKError::Io)?),
+                    AggOp::None => b.add_col_from    (&mat.partial_group_none(&g, 1).map_err(SKError::Io)?),
+                    AggOp::Sum  => b.add_col_from_int(&mat.partial_group_sum (&g).map_err(SKError::Io)?),
+                    AggOp::Min  => b.add_col_from_int(&mat.partial_group_min (&g).map_err(SKError::Io)?),
+                    AggOp::Max  => b.add_col_from_int(&mat.partial_group_max (&g).map_err(SKError::Io)?),
+                }.map_err(SKError::Io)?;
+            } else {
+                let b = dst_int.as_deref_mut().unwrap();
+                match spec.op {
+                    AggOp::Sum  => b.add_col_from    (&mat.partial_group_sum (&g).map_err(SKError::Io)?),
+                    AggOp::Min  => b.add_col_from    (&mat.partial_group_min (&g).map_err(SKError::Io)?),
+                    AggOp::Max  => b.add_col_from    (&mat.partial_group_max (&g).map_err(SKError::Io)?),
+                    AggOp::Any  => b.add_col_from_bit(&mat.partial_group_any (&g, 1).map_err(SKError::Io)?),
+                    AggOp::All  => b.add_col_from_bit(&mat.partial_group_all (&g, 1).map_err(SKError::Io)?),
+                    AggOp::None => b.add_col_from_bit(&mat.partial_group_none(&g, 1).map_err(SKError::Io)?),
+                }.map_err(SKError::Io)?;
             }
         }
     }
@@ -169,7 +142,7 @@ impl KmerPartition {
         src: &KmerPartition,
         i: usize,
         specs: &[OutputCol],
-        n_src_genomes: usize,
+        _n_src_genomes: usize,
         threshold: u32,
         output_presence: bool,
         in_place: bool,
@@ -189,7 +162,6 @@ impl KmerPartition {
             fs::create_dir_all(&dst_index_dir)?;
         }
 
-        let n_out = specs.len();
         let data_subdir = if output_presence { "presence" } else { "counts" };
 
         for l in 0..src_meta.n_layers {
@@ -202,7 +174,7 @@ impl KmerPartition {
             let presence_dir = src_layer_dir.join("presence");
             let src_is_count = counts_dir.exists() && !presence_dir.exists();
 
-            // Determine number of slots from the source matrix.
+            // Determine number of slots and detect implicit layers.
             let n = if counts_dir.exists() {
                 PersistentCompactIntMatrix::open(&src_layer_dir).map_err(SKError::Io)?.n()
             } else if presence_dir.exists() {
@@ -217,7 +189,7 @@ impl KmerPartition {
             };
 
             // Choose the output data directory (temp name for in-place).
-            let (dst_data_dir, final_data_dir) = if in_place {
+            let (dst_data_dir, final_data_dir): (PathBuf, PathBuf) = if in_place {
                 let tmp  = dst_layer_dir.join(format!("{data_subdir}_new"));
                 let perm = dst_layer_dir.join(data_subdir);
                 (tmp, perm)
@@ -232,37 +204,22 @@ impl KmerPartition {
             }
             fs::create_dir_all(&dst_data_dir)?;
 
-            // Initialise packed-format skeleton.
-            if output_presence {
-                PersistentBitMatrixBuilder::new(n, &dst_data_dir)
-                    .map_err(SKError::Io)?.close().map_err(SKError::Io)?;
+            let (mut dst_bit, mut dst_int) = if output_presence {
+                (Some(PersistentBitMatrixBuilder::new(n, &dst_data_dir).map_err(SKError::Io)?), None)
             } else {
-                PersistentCompactIntMatrixBuilder::new(n, &dst_data_dir)
-                    .map_err(SKError::Io)?.close().map_err(SKError::Io)?;
-            }
-
-            // Create column builders.
-            let mut builders: Vec<ColBuilder> = (0..n_out)
-                .map(|col| -> SKResult<ColBuilder> {
-                    if output_presence {
-                        Ok(ColBuilder::Bit(PersistentBitVecBuilder::new(
-                            n, &col_path_bit(&dst_data_dir, col),
-                        )?))
-                    } else {
-                        Ok(ColBuilder::Int(PersistentCompactIntVecBuilder::new(
-                            n, &col_path_int(&dst_data_dir, col),
-                        )?))
-                    }
-                })
-                .collect::<SKResult<_>>()?;
+                (None, Some(PersistentCompactIntMatrixBuilder::new(n, &dst_data_dir).map_err(SKError::Io)?))
+            };
 
             fill_builders(
-                &mut builders, specs, n, n_src_genomes,
-                &src_layer_dir, src_is_count, threshold,
+                specs, &src_layer_dir, src_is_count, threshold, output_presence,
+                dst_bit.as_mut(), dst_int.as_mut(),
             )?;
 
-            for b in builders { b.close()?; }
-            write_matrix_meta(&dst_data_dir, n, n_out).map_err(SKError::Io)?;
+            if output_presence {
+                dst_bit.unwrap().close().map_err(SKError::Io)?;
+            } else {
+                dst_int.unwrap().close().map_err(SKError::Io)?;
+            }
 
             // In-place: swap old data dir for new.
             if in_place {

src/obikpartitionner/src/tests/query_layer.rs

@@ -0,0 +1,79 @@
+use super::*;
+
+// ── QueryStats::AddAssign ───────────────────────────────────────────────────
+
+#[test]
+fn query_stats_add_assign_sums_fields() {
+    let mut total = QueryStats {
+        n_unique_kmers: 3,
+        n_mphf_calls: 5,
+        n_hits: 2,
+        n_columns_scanned: 1,
+        n_col_get_calls: 7,
+    };
+    total += QueryStats {
+        n_unique_kmers: 1,
+        n_mphf_calls: 4,
+        n_hits: 1,
+        n_columns_scanned: 2,
+        n_col_get_calls: 3,
+    };
+    assert_eq!(total.n_unique_kmers, 4);
+    assert_eq!(total.n_mphf_calls, 9);
+    assert_eq!(total.n_hits, 3);
+    assert_eq!(total.n_columns_scanned, 3);
+    assert_eq!(total.n_col_get_calls, 10);
+}
+
+#[test]
+fn query_stats_default_is_zero() {
+    let s = QueryStats::default();
+    assert_eq!(s.n_unique_kmers, 0);
+    assert_eq!(s.n_mphf_calls, 0);
+    assert_eq!(s.n_hits, 0);
+    assert_eq!(s.n_columns_scanned, 0);
+    assert_eq!(s.n_col_get_calls, 0);
+}
+
+// ── query_partition_with on a not-yet-indexed partition ─────────────────────
+
+/// A `KmerPartition` created but never taken through `build_layers` has no
+/// `index/` subdirectory under any partition — `query_partition_with` must
+/// recognise this and return default (all-zero) stats rather than erroring,
+/// exactly like an empty `kmers` map.
+#[test]
+fn query_partition_with_missing_index_dir_returns_default_stats() {
+    let tmp = tempfile::tempdir().expect("tempdir");
+    let partition = KmerPartition::create(tmp.path().join("idx"), 2, 21, 9, false)
+        .expect("create partition");
+
+    let mut kmers: HashMap<CanonicalKmer, Vec<KmerDesc>> = HashMap::new();
+    // Any well-formed canonical k-mer works here — the call must return
+    // before ever attempting an MPHF lookup, since `index/` doesn't exist.
+    let kmer = CanonicalKmer::from_raw_unchecked(0u64);
+    kmers.insert(kmer, vec![KmerDesc { seq_idx: 0, pos: 0 }]);
+
+    let stats = partition
+        .query_partition_with(0, &kmers, 1, false, |_event| {
+            panic!("on_event must not be called: no index was built");
+        })
+        .expect("query_partition_with should not error on a missing index dir");
+
+    assert_eq!(stats, QueryStats::default());
+}
+
+#[test]
+fn query_partition_with_empty_kmers_is_a_noop() {
+    let tmp = tempfile::tempdir().expect("tempdir");
+    let partition = KmerPartition::create(tmp.path().join("idx"), 2, 21, 9, false)
+        .expect("create partition");
+
+    let kmers: HashMap<CanonicalKmer, Vec<KmerDesc>> = HashMap::new();
+    let stats = partition
+        .query_partition_with(0, &kmers, 1, false, |_event| {
+            panic!("on_event must not be called on an empty kmer map");
+        })
+        .expect("query_partition_with on an empty map should not error");
+
+    assert_eq!(stats, QueryStats::default());
+}

src/obilayeredmap/src/layer.rs

@@ -6,7 +6,6 @@ use obicompactvec::{
     PersistentBitMatrix, PersistentBitMatrixBuilder,
     PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
 };
-use obicompactvec::traits::BitSliceMut;
 use obikseq::CanonicalKmer;
 use obiskio::{UnitigFileReader, UnitigFileWriter};
 
@@ -107,11 +106,7 @@ impl Layer<()> {
         let presence_dir = layer_dir.join(PRESENCE_DIR);
         fs::create_dir_all(&presence_dir).map_err(OLMError::Io)?;
         let mut mb = PersistentBitMatrixBuilder::new(n_kmers, &presence_dir).map_err(OLMError::Io)?;
-        let mut col = mb.add_col().map_err(OLMError::Io)?;
-        for slot in 0..n_kmers {
-            col.set(slot, true);
-        }
-        col.close().map_err(OLMError::Io)?;
+        mb.add_col_ones().map_err(OLMError::Io)?.close().map_err(OLMError::Io)?;
         mb.close().map_err(OLMError::Io)
     }
 }

src/obilayeredmap/src/layered_store.rs

@@ -96,163 +96,5 @@ impl<S: BitPartials> BitPartials for LayeredStore<S> {
 // ── Tests ─────────────────────────────────────────────────────────────────────
 
 #[cfg(test)]
-mod tests {
-    use super::*;
-    use obicompactvec::{
-        PersistentBitMatrix, PersistentBitMatrixBuilder,
-        PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
-    };
-    use obicompactvec::traits::BitSliceMut;
-    use tempfile::tempdir;
-
-    fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
-        let n = cols.first().map_or(0, |c| c.len());
-        let dir = tempdir().unwrap();
-        let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
-        for &col in cols {
-            let mut cb = b.add_col().unwrap();
-            for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
-            cb.close().unwrap();
-        }
-        b.close().unwrap();
-        let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
-        (dir, m)
-    }
-
-    fn make_bit_matrix(cols: &[&[bool]]) -> (tempfile::TempDir, PersistentBitMatrix) {
-        let n = cols.first().map_or(0, |c| c.len());
-        let dir = tempdir().unwrap();
-        let mut b = PersistentBitMatrixBuilder::new(n, &dir.path().join("presence")).unwrap();
-        for &col in cols {
-            let mut cb = b.add_col().unwrap();
-            for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
-            cb.close().unwrap();
-        }
-        b.close().unwrap();
-        let m = PersistentBitMatrix::open(dir.path()).unwrap();
-        (dir, m)
-    }
-
-    // ── ColumnWeights ─────────────────────────────────────────────────────────
-
-    #[test]
-    fn col_weights_sums_across_layers() {
-        // layer 0: col0=[1,2], col1=[3,4]   → weights [3, 7]
-        // layer 1: col0=[10,0], col1=[0,10]  → weights [10, 10]
-        // combined: [13, 17]
-        let (_d0, m0) = make_int_matrix(&[&[1, 2], &[3, 4]]);
-        let (_d1, m1) = make_int_matrix(&[&[10, 0], &[0, 10]]);
-        let store = LayeredStore::new(vec![m0, m1]);
-        let w = store.col_weights();
-        assert_eq!(w[0], 13);
-        assert_eq!(w[1], 17);
-    }
-
-    #[test]
-    fn col_weights_bit_sums_across_layers() {
-        // layer 0: col0=[T,F,T], col1=[F,T,T]  → counts [2, 2]
-        // layer 1: col0=[F,F,T], col1=[T,T,F]  → counts [1, 2]
-        // combined: [3, 4]
-        let (_d0, m0) = make_bit_matrix(&[&[true, false, true], &[false, true, true]]);
-        let (_d1, m1) = make_bit_matrix(&[&[false, false, true], &[true, true, false]]);
-        let store = LayeredStore::new(vec![m0, m1]);
-        let w = store.col_weights();
-        assert_eq!(w[0], 3);
-        assert_eq!(w[1], 4);
-    }
-
-    // ── CountPartials — layered (one partition) ───────────────────────────────
-
-    #[test]
-    fn layered_bray_matches_combined() {
-        // Split [1,2,3,4,5] across two layers; bray dist should equal direct computation
-        // on [1,2,3,4,5] for each column pair.
-        // col0=[1,2,3,4,5], col1=[5,4,3,2,1]
-        let (_d0, m0) = make_int_matrix(&[&[1, 2], &[5, 4]]);     // slots 0-1
-        let (_d1, m1) = make_int_matrix(&[&[3, 4, 5], &[3, 2, 1]]); // slots 2-4
-        let store = LayeredStore::new(vec![m0, m1]);
-
-        // direct on full data
-        let (_df, mf) = make_int_matrix(&[&[1, 2, 3, 4, 5], &[5, 4, 3, 2, 1]]);
-        let expected = CountPartials::bray_dist_matrix(&mf);
-        let got      = CountPartials::bray_dist_matrix(&store);
-        assert!((got[[0, 1]] - expected[[0, 1]]).abs() < 1e-12, "bray [0,1]");
-        assert!((got[[1, 0]] - expected[[1, 0]]).abs() < 1e-12, "bray [1,0]");
-    }
-
-    #[test]
-    fn layered_relfreq_bray_matches_combined() {
-        let (_d0, m0) = make_int_matrix(&[&[1, 2], &[5, 4]]);
-        let (_d1, m1) = make_int_matrix(&[&[3, 4, 5], &[3, 2, 1]]);
-        let store = LayeredStore::new(vec![m0, m1]);
-
-        let (_df, mf) = make_int_matrix(&[&[1, 2, 3, 4, 5], &[5, 4, 3, 2, 1]]);
-        let expected = CountPartials::relfreq_bray_dist_matrix(&mf);
-        let got      = CountPartials::relfreq_bray_dist_matrix(&store);
-        assert!((got[[0, 1]] - expected[[0, 1]]).abs() < 1e-12, "relfreq_bray [0,1]");
-    }
-
-    #[test]
-    fn layered_euclidean_matches_combined() {
-        let (_d0, m0) = make_int_matrix(&[&[3, 0], &[0, 4]]);
-        let (_d1, m1) = make_int_matrix(&[&[1, 1], &[2, 2]]);
-        let store = LayeredStore::new(vec![m0, m1]);
-
-        let (_df, mf) = make_int_matrix(&[&[3, 0, 1, 1], &[0, 4, 2, 2]]);
-        let expected = CountPartials::euclidean_dist_matrix(&mf);
-        let got      = CountPartials::euclidean_dist_matrix(&store);
-        assert!((got[[0, 1]] - expected[[0, 1]]).abs() < 1e-12, "euclidean [0,1]");
-    }
-
-    // ── CountPartials — partitioned (LayeredStore<LayeredStore<_>>) ───────────
-
-    #[test]
-    fn partitioned_bray_matches_combined() {
-        // partition 0: slots [1,2,3,4,5]  col0 vs col1
-        // partition 1: slots [10,20]      col0 vs col1
-        let (_d0, p0) = make_int_matrix(&[&[1, 2, 3, 4, 5], &[5, 4, 3, 2, 1]]);
-        let (_d1, p1) = make_int_matrix(&[&[10, 20], &[20, 10]]);
-
-        let partitioned = LayeredStore::new(vec![
-            LayeredStore::new(vec![p0]),
-            LayeredStore::new(vec![p1]),
-        ]);
-
-        let (_df, mf) = make_int_matrix(&[&[1, 2, 3, 4, 5, 10, 20], &[5, 4, 3, 2, 1, 20, 10]]);
-        let expected = CountPartials::bray_dist_matrix(&mf);
-        let got      = CountPartials::bray_dist_matrix(&partitioned);
-        assert!((got[[0, 1]] - expected[[0, 1]]).abs() < 1e-12, "partitioned bray [0,1]");
-    }
-
-    // ── BitPartials ───────────────────────────────────────────────────────────
-
-    #[test]
-    fn layered_jaccard_matches_combined() {
-        let (_d0, m0) = make_bit_matrix(&[&[true, false], &[false, true]]);
-        let (_d1, m1) = make_bit_matrix(&[&[true, true], &[true, false]]);
-        let store = LayeredStore::new(vec![m0, m1]);
-
-        let (_df, mf) = make_bit_matrix(&[
-            &[true, false, true, true],
-            &[false, true, true, false],
-        ]);
-        let expected = BitPartials::jaccard_dist_matrix(&mf);
-        let got      = BitPartials::jaccard_dist_matrix(&store);
-        assert!((got[[0, 1]] - expected[[0, 1]]).abs() < 1e-12, "jaccard [0,1]");
-    }
-
-    #[test]
-    fn layered_hamming_matches_combined() {
-        let (_d0, m0) = make_bit_matrix(&[&[true, false], &[false, true]]);
-        let (_d1, m1) = make_bit_matrix(&[&[true, true], &[false, false]]);
-        let store = LayeredStore::new(vec![m0, m1]);
-
-        let (_df, mf) = make_bit_matrix(&[
-            &[true, false, true, true],
-            &[false, true, false, false],
-        ]);
-        let expected = BitPartials::hamming_dist_matrix(&mf);
-        let got      = BitPartials::hamming_dist_matrix(&store);
-        assert_eq!(got[[0, 1]], expected[[0, 1]], "hamming [0,1]");
-    }
-}
+#[path = "tests/layered_store.rs"]
+mod tests;

src/obilayeredmap/src/tests/layered_store.rs

@@ -0,0 +1,381 @@
+use super::*;
+use obicompactvec::{
+    PersistentBitMatrix, PersistentBitMatrixBuilder,
+    PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
+};
+use tempfile::tempdir;
+
+fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
+    let n = cols.first().map_or(0, |c| c.len());
+    let dir = tempdir().unwrap();
+    let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
+    for &col in cols {
+        let mut cb = b.add_col().unwrap();
+        for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
+        cb.close().unwrap();
+    }
+    b.close().unwrap();
+    let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
+    (dir, m)
+}
+
+fn make_bit_matrix(cols: &[&[bool]]) -> (tempfile::TempDir, PersistentBitMatrix) {
+    let n = cols.first().map_or(0, |c| c.len());
+    let dir = tempdir().unwrap();
+    let mut b = PersistentBitMatrixBuilder::new(n, &dir.path().join("presence")).unwrap();
+    for &col in cols {
+        let mut cb = b.add_col().unwrap();
+        for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
+        cb.close().unwrap();
+    }
+    b.close().unwrap();
+    let m = PersistentBitMatrix::open(dir.path()).unwrap();
+    (dir, m)
+}
+
+// ── ColumnWeights ─────────────────────────────────────────────────────────
+
+#[test]
+fn col_weights_sums_across_layers() {
+    // layer 0: col0=[1,2], col1=[3,4]   → weights [3, 7]
+    // layer 1: col0=[10,0], col1=[0,10]  → weights [10, 10]
+    // combined: [13, 17]
+    let (_d0, m0) = make_int_matrix(&[&[1, 2], &[3, 4]]);
+    let (_d1, m1) = make_int_matrix(&[&[10, 0], &[0, 10]]);
+    let store = LayeredStore::new(vec![m0, m1]);
+    let w = store.col_weights();
+    assert_eq!(w[0], 13);
+    assert_eq!(w[1], 17);
+}
+
+#[test]
+fn col_weights_bit_sums_across_layers() {
+    // layer 0: col0=[T,F,T], col1=[F,T,T]  → counts [2, 2]
+    // layer 1: col0=[F,F,T], col1=[T,T,F]  → counts [1, 2]
+    // combined: [3, 4]
+    let (_d0, m0) = make_bit_matrix(&[&[true, false, true], &[false, true, true]]);
+    let (_d1, m1) = make_bit_matrix(&[&[false, false, true], &[true, true, false]]);
+    let store = LayeredStore::new(vec![m0, m1]);
+    let w = store.col_weights();
+    assert_eq!(w[0], 3);
+    assert_eq!(w[1], 4);
+}
+
+// ── CountPartials — layered (one partition) ───────────────────────────────
+
+#[test]
+fn layered_bray_matches_combined() {
+    // Split [1,2,3,4,5] across two layers; bray dist should equal direct computation
+    // on [1,2,3,4,5] for each column pair.
+    // col0=[1,2,3,4,5], col1=[5,4,3,2,1]
+    let (_d0, m0) = make_int_matrix(&[&[1, 2], &[5, 4]]);     // slots 0-1
+    let (_d1, m1) = make_int_matrix(&[&[3, 4, 5], &[3, 2, 1]]); // slots 2-4
+    let store = LayeredStore::new(vec![m0, m1]);
+
+    // direct on full data
+    let (_df, mf) = make_int_matrix(&[&[1, 2, 3, 4, 5], &[5, 4, 3, 2, 1]]);
+    let expected = CountPartials::bray_dist_matrix(&mf);
+    let got      = CountPartials::bray_dist_matrix(&store);
+    assert!((got[[0, 1]] - expected[[0, 1]]).abs() < 1e-12, "bray [0,1]");
+    assert!((got[[1, 0]] - expected[[1, 0]]).abs() < 1e-12, "bray [1,0]");
+}
+
+#[test]
+fn layered_relfreq_bray_matches_combined() {
+    let (_d0, m0) = make_int_matrix(&[&[1, 2], &[5, 4]]);
+    let (_d1, m1) = make_int_matrix(&[&[3, 4, 5], &[3, 2, 1]]);
+    let store = LayeredStore::new(vec![m0, m1]);
+
+    let (_df, mf) = make_int_matrix(&[&[1, 2, 3, 4, 5], &[5, 4, 3, 2, 1]]);
+    let expected = CountPartials::relfreq_bray_dist_matrix(&mf);
+    let got      = CountPartials::relfreq_bray_dist_matrix(&store);
+    assert!((got[[0, 1]] - expected[[0, 1]]).abs() < 1e-12, "relfreq_bray [0,1]");
+}
+
+#[test]
+fn layered_euclidean_matches_combined() {
+    let (_d0, m0) = make_int_matrix(&[&[3, 0], &[0, 4]]);
+    let (_d1, m1) = make_int_matrix(&[&[1, 1], &[2, 2]]);
+    let store = LayeredStore::new(vec![m0, m1]);
+
+    let (_df, mf) = make_int_matrix(&[&[3, 0, 1, 1], &[0, 4, 2, 2]]);
+    let expected = CountPartials::euclidean_dist_matrix(&mf);
+    let got      = CountPartials::euclidean_dist_matrix(&store);
+    assert!((got[[0, 1]] - expected[[0, 1]]).abs() < 1e-12, "euclidean [0,1]");
+}
+
+// ── CountPartials — partitioned (LayeredStore<LayeredStore<_>>) ───────────
+
+#[test]
+fn partitioned_bray_matches_combined() {
+    // partition 0: slots [1,2,3,4,5]  col0 vs col1
+    // partition 1: slots [10,20]      col0 vs col1
+    let (_d0, p0) = make_int_matrix(&[&[1, 2, 3, 4, 5], &[5, 4, 3, 2, 1]]);
+    let (_d1, p1) = make_int_matrix(&[&[10, 20], &[20, 10]]);
+
+    let partitioned = LayeredStore::new(vec![
+        LayeredStore::new(vec![p0]),
+        LayeredStore::new(vec![p1]),
+    ]);
+
+    let (_df, mf) = make_int_matrix(&[&[1, 2, 3, 4, 5, 10, 20], &[5, 4, 3, 2, 1, 20, 10]]);
+    let expected = CountPartials::bray_dist_matrix(&mf);
+    let got      = CountPartials::bray_dist_matrix(&partitioned);
+    assert!((got[[0, 1]] - expected[[0, 1]]).abs() < 1e-12, "partitioned bray [0,1]");
+}
+
+#[test]
+fn partitioned_threshold_jaccard_off_diagonal_is_pairwise() {
+    // 3 genomes, 2 partitions, 1 layer each — mirrors distance.rs's
+    // LayeredStore<LayeredStore<PersistentCompactIntMatrix>> shape.
+    // partition 0: col0=[3,0], col1=[0,3], col2=[3,3]
+    // partition 1: col0=[1,1], col1=[1,0], col2=[0,1]
+    let (_d0, p0) = make_int_matrix(&[&[3, 0], &[0, 3], &[3, 3]]);
+    let (_d1, p1) = make_int_matrix(&[&[1, 1], &[1, 0], &[0, 1]]);
+
+    let partitioned = LayeredStore::new(vec![
+        LayeredStore::new(vec![p0]),
+        LayeredStore::new(vec![p1]),
+    ]);
+
+    let (_df, mf) = make_int_matrix(&[&[3, 0, 1, 1], &[0, 3, 1, 0], &[3, 3, 0, 1]]);
+    let threshold = 1u32;
+    let (inter_p, union_p) = CountPartials::partial_threshold_jaccard(&partitioned, threshold);
+    let (inter_f, union_f) = CountPartials::partial_threshold_jaccard(&mf, threshold);
+
+    let n = 3;
+    for i in 0..n {
+        for j in 0..n {
+            assert_eq!(inter_p[[i, j]], inter_f[[i, j]], "inter[{i},{j}]");
+            assert_eq!(union_p[[i, j]], union_f[[i, j]], "union[{i},{j}]");
+        }
+    }
+}
+
+#[test]
+fn partitioned_threshold_jaccard_packed_off_diagonal_is_pairwise() {
+    // Same as `partitioned_threshold_jaccard_off_diagonal_is_pairwise` but
+    // each partition matrix is packed into a single .pcmx file first —
+    // the on-disk format actually used in production after `pack_matrices`.
+    use obicompactvec::pack_compact_int_matrix;
+
+    let (d0, _p0) = make_int_matrix(&[&[3, 0], &[0, 3], &[3, 3]]);
+    pack_compact_int_matrix(&d0.path().join("counts")).unwrap();
+    let p0 = PersistentCompactIntMatrix::open(d0.path()).unwrap();
+
+    let (d1, _p1) = make_int_matrix(&[&[1, 1], &[1, 0], &[0, 1]]);
+    pack_compact_int_matrix(&d1.path().join("counts")).unwrap();
+    let p1 = PersistentCompactIntMatrix::open(d1.path()).unwrap();
+
+    let partitioned = LayeredStore::new(vec![
+        LayeredStore::new(vec![p0]),
+        LayeredStore::new(vec![p1]),
+    ]);
+
+    let (_df, mf) = make_int_matrix(&[&[3, 0, 1, 1], &[0, 3, 1, 0], &[3, 3, 0, 1]]);
+    let threshold = 1u32;
+    let (inter_p, union_p) = CountPartials::partial_threshold_jaccard(&partitioned, threshold);
+    let (inter_f, union_f) = CountPartials::partial_threshold_jaccard(&mf, threshold);
+
+    let n = 3;
+    for i in 0..n {
+        for j in 0..n {
+            assert_eq!(inter_p[[i, j]], inter_f[[i, j]], "inter[{i},{j}]");
+            assert_eq!(union_p[[i, j]], union_f[[i, j]], "union[{i},{j}]");
+        }
+    }
+}
+
+#[test]
+fn partitioned_multilayer_threshold_jaccard_off_diagonal_is_pairwise() {
+    // 2 partitions, 2 layers each — the shape production indexes actually
+    // have (MPHF collision layers within a partition).
+    // partition 0, layer 0: col0=[3,0], col1=[0,3], col2=[3,3]
+    // partition 0, layer 1: col0=[2,0], col1=[0,0], col2=[2,0]
+    // partition 1, layer 0: col0=[1,1], col1=[1,0], col2=[0,1]
+    // partition 1, layer 1: col0=[0,5], col1=[5,5], col2=[0,0]
+    let (_d0a, p0a) = make_int_matrix(&[&[3, 0], &[0, 3], &[3, 3]]);
+    let (_d0b, p0b) = make_int_matrix(&[&[2, 0], &[0, 0], &[2, 0]]);
+    let (_d1a, p1a) = make_int_matrix(&[&[1, 1], &[1, 0], &[0, 1]]);
+    let (_d1b, p1b) = make_int_matrix(&[&[0, 5], &[5, 5], &[0, 0]]);
+
+    let partitioned = LayeredStore::new(vec![
+        LayeredStore::new(vec![p0a, p0b]),
+        LayeredStore::new(vec![p1a, p1b]),
+    ]);
+
+    // Flattened equivalent: concatenate every layer's slots into one matrix.
+    let (_df, mf) = make_int_matrix(&[
+        &[3, 0, 2, 0, 1, 1, 0, 5],
+        &[0, 3, 0, 0, 1, 0, 5, 5],
+        &[3, 3, 2, 0, 0, 1, 0, 0],
+    ]);
+    let threshold = 1u32;
+    let (inter_p, union_p) = CountPartials::partial_threshold_jaccard(&partitioned, threshold);
+    let (inter_f, union_f) = CountPartials::partial_threshold_jaccard(&mf, threshold);
+
+    let n = 3;
+    for i in 0..n {
+        for j in 0..n {
+            assert_eq!(inter_p[[i, j]], inter_f[[i, j]], "inter[{i},{j}]");
+            assert_eq!(union_p[[i, j]], union_f[[i, j]], "union[{i},{j}]");
+        }
+    }
+}
+
+// ── BitPartials ───────────────────────────────────────────────────────────
+
+#[test]
+fn layered_jaccard_matches_combined() {
+    let (_d0, m0) = make_bit_matrix(&[&[true, false], &[false, true]]);
+    let (_d1, m1) = make_bit_matrix(&[&[true, true], &[true, false]]);
+    let store = LayeredStore::new(vec![m0, m1]);
+
+    let (_df, mf) = make_bit_matrix(&[
+        &[true, false, true, true],
+        &[false, true, true, false],
+    ]);
+    let expected = BitPartials::jaccard_dist_matrix(&mf);
+    let got      = BitPartials::jaccard_dist_matrix(&store);
+    assert!((got[[0, 1]] - expected[[0, 1]]).abs() < 1e-12, "jaccard [0,1]");
+}
+
+#[test]
+fn layered_hamming_matches_combined() {
+    let (_d0, m0) = make_bit_matrix(&[&[true, false], &[false, true]]);
+    let (_d1, m1) = make_bit_matrix(&[&[true, true], &[false, false]]);
+    let store = LayeredStore::new(vec![m0, m1]);
+
+    let (_df, mf) = make_bit_matrix(&[
+        &[true, false, true, true],
+        &[false, true, false, false],
+    ]);
+    let expected = BitPartials::hamming_dist_matrix(&mf);
+    let got      = BitPartials::hamming_dist_matrix(&store);
+    assert_eq!(got[[0, 1]], expected[[0, 1]], "hamming [0,1]");
+}
+
+#[test]
+fn partitioned_bit_jaccard_off_diagonal_is_pairwise() {
+    // Same shape as the count-based `partitioned_multilayer_threshold_jaccard_*`
+    // tests, but for the presence/bit path (`with_counts = false` — what
+    // `all_specifics` actually uses in production).
+    // 4 genomes, 3 partitions, 2 layers in the last one.
+    let (_d0, p0) = make_bit_matrix(&[
+        &[true, false, true],
+        &[false, true, true],
+        &[true, true, false],
+        &[false, false, true],
+    ]);
+    let (_d1, p1) = make_bit_matrix(&[
+        &[true, true],
+        &[false, true],
+        &[true, false],
+        &[true, true],
+    ]);
+    let (_d2a, p2a) = make_bit_matrix(&[
+        &[false, true],
+        &[true, true],
+        &[false, false],
+        &[true, false],
+    ]);
+    let (_d2b, p2b) = make_bit_matrix(&[
+        &[true],
+        &[false],
+        &[true],
+        &[true],
+    ]);
+
+    let partitioned = LayeredStore::new(vec![
+        LayeredStore::new(vec![p0]),
+        LayeredStore::new(vec![p1]),
+        LayeredStore::new(vec![p2a, p2b]),
+    ]);
+
+    // Flattened equivalent: concatenate every partition/layer's slots.
+    let (_df, mf) = make_bit_matrix(&[
+        &[true, false, true, true, true, false, true, true],
+        &[false, true, true, false, true, true, true, false],
+        &[true, true, false, true, false, false, false, true],
+        &[false, false, true, true, true, true, false, true],
+    ]);
+
+    let (inter_p, union_p) = BitPartials::partial_jaccard(&partitioned);
+    let (inter_f, union_f) = BitPartials::partial_jaccard(&mf);
+
+    let n = 4;
+    for i in 0..n {
+        for j in 0..n {
+            assert_eq!(inter_p[[i, j]], inter_f[[i, j]], "inter[{i},{j}]");
+            assert_eq!(union_p[[i, j]], union_f[[i, j]], "union[{i},{j}]");
+        }
+    }
+}
+
+#[test]
+fn partitioned_bit_jaccard_packed_off_diagonal_is_pairwise() {
+    // Same as `partitioned_bit_jaccard_off_diagonal_is_pairwise` but every
+    // partition's presence matrix is packed into a single .pbmx file —
+    // the on-disk format actually used in production after `pack_matrices`.
+    use obicompactvec::pack_bit_matrix;
+
+    let (d0, _p0) = make_bit_matrix(&[
+        &[true, false, true],
+        &[false, true, true],
+        &[true, true, false],
+        &[false, false, true],
+    ]);
+    pack_bit_matrix(&d0.path().join("presence")).unwrap();
+    let p0 = PersistentBitMatrix::open(d0.path()).unwrap();
+
+    let (d1, _p1) = make_bit_matrix(&[
+        &[true, true],
+        &[false, true],
+        &[true, false],
+        &[true, true],
+    ]);
+    pack_bit_matrix(&d1.path().join("presence")).unwrap();
+    let p1 = PersistentBitMatrix::open(d1.path()).unwrap();
+
+    let (d2a, _p2a) = make_bit_matrix(&[
+        &[false, true],
+        &[true, true],
+        &[false, false],
+        &[true, false],
+    ]);
+    pack_bit_matrix(&d2a.path().join("presence")).unwrap();
+    let p2a = PersistentBitMatrix::open(d2a.path()).unwrap();
+
+    let (d2b, _p2b) = make_bit_matrix(&[
+        &[true],
+        &[false],
+        &[true],
+        &[true],
+    ]);
+    pack_bit_matrix(&d2b.path().join("presence")).unwrap();
+    let p2b = PersistentBitMatrix::open(d2b.path()).unwrap();
+
+    let partitioned = LayeredStore::new(vec![
+        LayeredStore::new(vec![p0]),
+        LayeredStore::new(vec![p1]),
+        LayeredStore::new(vec![p2a, p2b]),
+    ]);
+
+    let (_df, mf) = make_bit_matrix(&[
+        &[true, false, true, true, true, false, true, true],
+        &[false, true, true, false, true, true, true, false],
+        &[true, true, false, true, false, false, false, true],
+        &[false, false, true, true, true, true, false, true],
+    ]);
+
+    let (inter_p, union_p) = BitPartials::partial_jaccard(&partitioned);
+    let (inter_f, union_f) = BitPartials::partial_jaccard(&mf);
+
+    let n = 4;
+    for i in 0..n {
+        for j in 0..n {
+            assert_eq!(inter_p[[i, j]], inter_f[[i, j]], "inter[{i},{j}]");
+            assert_eq!(union_p[[i, j]], union_f[[i, j]], "union[{i},{j}]");
+        }
+    }
+}

src/obisys/src/lib.rs

@@ -4,7 +4,7 @@ use std::sync::{Condvar, Mutex};
 use std::time::{Duration, Instant};
 
 use indicatif::{ProgressBar, ProgressStyle};
-use tracing::{info, warn};
+use tracing::{debug, info, warn};
 
 const BRAILLE: &[&str] = &["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
 
@@ -14,24 +14,25 @@ const BRAILLE: &[&str] = &["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧
 /// a TTY (e.g. HPC job logs): every 10% for bounded bars, every ~10 s for
 /// spinners (throttled on `set_message`).
 pub struct TracedBar {
-    pb:          ProgressBar,
-    label:       String,
-    unit:        String,
-    total:       u64,        // 0 for spinners
-    start:       Instant,    // creation time, for spinner throttling
-    last_pct:    AtomicU64,  // last emitted 10%-bucket (1..=10), 0 = none yet
-    last_log_ms: AtomicU64,  // ms since `start` at last spinner log
+    pb: ProgressBar,
+    label: String,
+    unit: String,
+    total: u64,             // 0 for spinners
+    start: Instant,         // creation time, for spinner throttling
+    last_pct: AtomicU64,    // last emitted 10%-bucket (1..=10), 0 = none yet
+    last_log_ms: AtomicU64, // ms since `start` at last spinner log
 }
 
 impl TracedBar {
     pub fn inc(&self, delta: u64) {
         self.pb.inc(delta);
         if self.pb.is_hidden() && self.total > 0 {
-            let pos   = self.pb.position();
+            let pos = self.pb.position();
             let pct10 = (pos * 10) / self.total; // 0..=10
-            let last  = self.last_pct.load(Ordering::Relaxed);
+            let last = self.last_pct.load(Ordering::Relaxed);
             if pct10 > last
-                && self.last_pct
+                && self
+                    .last_pct
                     .compare_exchange(last, pct10, Ordering::Relaxed, Ordering::Relaxed)
                     .is_ok()
             {
@@ -49,14 +50,14 @@ impl TracedBar {
         let msg = msg.into();
         if self.pb.is_hidden() {
             if self.total > 0 {
-                // bounded bar: always log (already rate-limited by 10% threshold in inc)
-                info!(stage = %self.label, "{msg}");
+                debug!(stage = %self.label, "{msg}");
             } else {
                 // spinner: throttle to ~10 s
                 let now_ms = self.start.elapsed().as_millis() as u64;
-                let last   = self.last_log_ms.load(Ordering::Relaxed);
+                let last = self.last_log_ms.load(Ordering::Relaxed);
                 if now_ms >= last + 10_000
-                    && self.last_log_ms
+                    && self
+                        .last_log_ms
                         .compare_exchange(last, now_ms, Ordering::Relaxed, Ordering::Relaxed)
                         .is_ok()
                 {
@@ -83,8 +84,13 @@ pub fn spinner(label: &str) -> TracedBar {
     );
     pb.enable_steady_tick(Duration::from_millis(100));
     TracedBar {
-        pb, label: label.to_string(), unit: String::new(), total: 0,
-        start: Instant::now(), last_pct: AtomicU64::new(0), last_log_ms: AtomicU64::new(0),
+        pb,
+        label: label.to_string(),
+        unit: String::new(),
+        total: 0,
+        start: Instant::now(),
+        last_pct: AtomicU64::new(0),
+        last_log_ms: AtomicU64::new(0),
     }
 }
 
@@ -101,8 +107,13 @@ pub fn progress_bar(label: &str, n: u64, unit: &str) -> TracedBar {
     );
     pb.enable_steady_tick(Duration::from_millis(100));
     TracedBar {
-        pb, label: label.to_string(), unit: unit.to_string(), total: n,
-        start: Instant::now(), last_pct: AtomicU64::new(0), last_log_ms: AtomicU64::new(0),
+        pb,
+        label: label.to_string(),
+        unit: unit.to_string(),
+        total: n,
+        start: Instant::now(),
+        last_pct: AtomicU64::new(0),
+        last_log_ms: AtomicU64::new(0),
     }
 }
 
@@ -204,13 +215,19 @@ fn tv_to_secs(tv: timeval) -> f64 {
 }
 
 #[cfg(target_os = "macos")]
-fn rss_to_bytes(ru: &rusage) -> u64 { ru.ru_maxrss as u64 }
+fn rss_to_bytes(ru: &rusage) -> u64 {
+    ru.ru_maxrss as u64
+}
 
 #[cfg(not(target_os = "macos"))]
-fn rss_to_bytes(ru: &rusage) -> u64 { ru.ru_maxrss as u64 * 1024 }
+fn rss_to_bytes(ru: &rusage) -> u64 {
+    ru.ru_maxrss as u64 * 1024
+}
 
 // Monotonically increasing counters — negative delta would be a kernel bug.
-fn delta(end: i64, start: i64) -> u64 { (end - start).max(0) as u64 }
+fn delta(end: i64, start: i64) -> u64 {
+    (end - start).max(0) as u64
+}
 
 // ── CpuSample ─────────────────────────────────────────────────────────────────
 
@@ -218,31 +235,151 @@ fn delta(end: i64, start: i64) -> u64 { (end - start).max(0) as u64 }
 /// Use [`cpu_efficiency`](Self::cpu_efficiency) to measure the fraction of
 /// available cores used since the snapshot was taken.
 pub struct CpuSample {
-    wall:      Instant,
+    wall: Instant,
     user_secs: f64,
-    sys_secs:  f64,
+    sys_secs: f64,
+    previous: f64,
 }
 
 impl CpuSample {
     pub fn now() -> Self {
         let ru = get_rusage();
         Self {
-            wall:      Instant::now(),
+            wall: Instant::now(),
             user_secs: tv_to_secs(ru.ru_utime),
-            sys_secs:  tv_to_secs(ru.ru_stime),
+            sys_secs: tv_to_secs(ru.ru_stime),
+            previous: 0.0,
         }
     }
 
     /// (user_delta + sys_delta) / (wall_delta × n_cores) since this snapshot.
     /// Returns 0.0 if less than 100 ms have elapsed (too noisy).
     pub fn cpu_efficiency(&self, n_cores: usize) -> f64 {
-        let ru   = get_rusage();
+        let ru = get_rusage();
         let wall = self.wall.elapsed().as_secs_f64();
-        if wall < 0.1 { return 0.0; }
-        let cpu = (tv_to_secs(ru.ru_utime) - self.user_secs)
-                + (tv_to_secs(ru.ru_stime) - self.sys_secs);
+        if wall < 0.1 {
+            return 0.0;
+        }
+        let cpu =
+            (tv_to_secs(ru.ru_utime) - self.user_secs) + (tv_to_secs(ru.ru_stime) - self.sys_secs);
         cpu / (wall * n_cores as f64)
     }
+
+    pub fn do_i_activate(&mut self, threshold: f64) -> bool {
+        let delta_wall = self.wall.elapsed().as_secs_f64();
+        if delta_wall < 0.1 {
+            // Window too short to be meaningful — leave state untouched so it
+            // keeps accumulating until a real sample can be taken.
+            return false;
+        }
+
+        let n = CpuSample::now();
+        let delta_ru = (n.user_secs - self.user_secs) + (n.sys_secs - self.sys_secs);
+
+        let efficiency = delta_ru / delta_wall;
+        let activate = 0f64.max(efficiency - self.previous) >= threshold;
+
+        debug!(
+            "Do I activate : {} -> {} = {} Activate: {}",
+            self.previous,
+            efficiency,
+            0f64.max(efficiency - self.previous),
+            activate
+        );
+        self.previous = efficiency;
+        self.user_secs = n.user_secs;
+        self.sys_secs = n.sys_secs;
+        self.wall = n.wall;
+
+        activate
+    }
+}
+
+// ── IoSample ──────────────────────────────────────────────────────────────────
+
+/// Snapshot of process-wide block I/O (bytes read + written) + wall clock.
+///
+/// Same activation protocol as [`CpuSample`], but the growth check in
+/// [`do_i_activate`](Self::do_i_activate) is *relative* rather than absolute:
+/// raw I/O throughput has no portable scale across storage devices, unlike a
+/// core count.
+pub struct IoSample {
+    wall: Instant,
+    bytes: u64,
+    previous_rate: f64,
+}
+
+impl IoSample {
+    pub fn now() -> Self {
+        Self {
+            wall: Instant::now(),
+            bytes: Self::read_bytes(),
+            previous_rate: 0.0,
+        }
+    }
+
+    /// Bytes actually submitted to the block layer (read + write), summed
+    /// process-wide. Returns 0 if unavailable — degrades gracefully to a
+    /// signal that never triggers activation (CPU-only heuristic).
+    #[cfg(target_os = "linux")]
+    fn read_bytes() -> u64 {
+        let Ok(io) = std::fs::read_to_string("/proc/self/io") else {
+            return 0;
+        };
+        io.lines()
+            .filter_map(|l| {
+                l.strip_prefix("read_bytes: ")
+                    .or_else(|| l.strip_prefix("write_bytes: "))
+            })
+            .filter_map(|v| v.trim().parse::<u64>().ok())
+            .sum()
+    }
+
+    #[cfg(target_os = "macos")]
+    fn read_bytes() -> u64 {
+        use libc::{RUSAGE_INFO_V4, getpid, proc_pid_rusage, rusage_info_v4};
+        let mut info: rusage_info_v4 = unsafe { std::mem::zeroed() };
+        let ret =
+            unsafe { proc_pid_rusage(getpid(), RUSAGE_INFO_V4, &mut info as *mut _ as *mut _) };
+        if ret != 0 {
+            return 0;
+        }
+        info.ri_diskio_bytesread + info.ri_diskio_byteswritten
+    }
+
+    #[cfg(not(any(target_os = "linux", target_os = "macos")))]
+    fn read_bytes() -> u64 {
+        0
+    }
+
+    /// Same protocol as [`CpuSample::do_i_activate`] (0.1 s minimum window,
+    /// state untouched on early return), but growth is measured relative to
+    /// the previous rate. `threshold` is a fraction, e.g. `0.2` for a 20 %
+    /// increase in throughput since the last real sample.
+    pub fn do_i_activate(&mut self, threshold: f64) -> bool {
+        let elapsed = self.wall.elapsed().as_secs_f64();
+        if elapsed < 0.1 {
+            return false;
+        }
+
+        let n = Self::read_bytes();
+        let rate = n.saturating_sub(self.bytes) as f64 / elapsed;
+        let activate = if self.previous_rate == 0.0 {
+            rate > 0.0 // bootstrap: any measured throughput is signal enough
+        } else {
+            (rate - self.previous_rate) / self.previous_rate >= threshold
+        };
+
+        debug!(
+            "Do I activate (I/O) : {} -> {} Activate: {}",
+            self.previous_rate, rate, activate
+        );
+        self.previous_rate = rate;
+        self.bytes = n;
+        self.wall = Instant::now();
+
+        activate
+    }
 }
 
 // ── public API ────────────────────────────────────────────────────────────────
@@ -251,33 +388,37 @@ impl CpuSample {
 #[must_use = "call .stop() to record the stage"]
 pub struct Stage {
     label: String,
-    wall:  Instant,
-    ru:    rusage,
+    wall: Instant,
+    ru: rusage,
 }
 
 impl Stage {
     pub fn start(label: impl Into<String>) -> Self {
         let label = label.into();
         info!(stage = %label, "started");
-        Self { label, wall: Instant::now(), ru: get_rusage() }
+        Self {
+            label,
+            wall: Instant::now(),
+            ru: get_rusage(),
+        }
     }
 
     pub fn stop(self) -> StageStats {
         let wall_secs = self.wall.elapsed().as_secs_f64();
         let end = get_rusage();
         let stats = StageStats {
-            label:         self.label,
+            label: self.label,
             wall_secs,
-            user_secs:     tv_to_secs(end.ru_utime) - tv_to_secs(self.ru.ru_utime),
-            sys_secs:      tv_to_secs(end.ru_stime) - tv_to_secs(self.ru.ru_stime),
+            user_secs: tv_to_secs(end.ru_utime) - tv_to_secs(self.ru.ru_utime),
+            sys_secs: tv_to_secs(end.ru_stime) - tv_to_secs(self.ru.ru_stime),
             max_rss_bytes: rss_to_bytes(&end),
-            minor_faults:  delta(end.ru_minflt  as i64, self.ru.ru_minflt  as i64),
-            major_faults:  delta(end.ru_majflt  as i64, self.ru.ru_majflt  as i64),
-            vol_ctx:       delta(end.ru_nvcsw   as i64, self.ru.ru_nvcsw   as i64),
-            invol_ctx:     delta(end.ru_nivcsw  as i64, self.ru.ru_nivcsw  as i64),
-            in_blocks:     delta(end.ru_inblock as i64, self.ru.ru_inblock as i64),
-            out_blocks:    delta(end.ru_oublock as i64, self.ru.ru_oublock as i64),
-            swaps:         delta(end.ru_nswap   as i64, self.ru.ru_nswap   as i64),
+            minor_faults: delta(end.ru_minflt as i64, self.ru.ru_minflt as i64),
+            major_faults: delta(end.ru_majflt as i64, self.ru.ru_majflt as i64),
+            vol_ctx: delta(end.ru_nvcsw as i64, self.ru.ru_nvcsw as i64),
+            invol_ctx: delta(end.ru_nivcsw as i64, self.ru.ru_nivcsw as i64),
+            in_blocks: delta(end.ru_inblock as i64, self.ru.ru_inblock as i64),
+            out_blocks: delta(end.ru_oublock as i64, self.ru.ru_oublock as i64),
+            swaps: delta(end.ru_nswap as i64, self.ru.ru_nswap as i64),
         };
         info!(
             stage      = %stats.label,
@@ -299,27 +440,30 @@ impl Stage {
 
 /// Per-stage efficiency metrics collected from `getrusage(RUSAGE_SELF)` deltas.
 pub struct StageStats {
-    pub label:         String,
-    pub wall_secs:     f64,
-    pub user_secs:     f64,
-    pub sys_secs:      f64,
+    pub label: String,
+    pub wall_secs: f64,
+    pub user_secs: f64,
+    pub sys_secs: f64,
     /// Peak RSS at end of stage (bytes). ru_maxrss is a process-lifetime maximum,
     /// so this reflects the high-water mark up to and including this stage.
     pub max_rss_bytes: u64,
-    pub minor_faults:  u64,
-    pub major_faults:  u64,
-    pub vol_ctx:       u64,   // voluntary context switches
-    pub invol_ctx:     u64,   // involuntary context switches
-    pub in_blocks:     u64,   // filesystem block reads (after page cache)
-    pub out_blocks:    u64,   // filesystem block writes
-    pub swaps:         u64,
+    pub minor_faults: u64,
+    pub major_faults: u64,
+    pub vol_ctx: u64,    // voluntary context switches
+    pub invol_ctx: u64,  // involuntary context switches
+    pub in_blocks: u64,  // filesystem block reads (after page cache)
+    pub out_blocks: u64, // filesystem block writes
+    pub swaps: u64,
 }
 
 impl StageStats {
     /// (user + sys) / wall — effective thread count utilisation.
     pub fn parallelism(&self) -> f64 {
-        if self.wall_secs > 1e-9 { (self.user_secs + self.sys_secs) / self.wall_secs }
-        else { 0.0 }
+        if self.wall_secs > 1e-9 {
+            (self.user_secs + self.sys_secs) / self.wall_secs
+        } else {
+            0.0
+        }
     }
 
     /// parallelism / n_cores — fraction of available CPU power used (0..1+).
@@ -335,25 +479,33 @@ pub struct Reporter {
 }
 
 impl Reporter {
-    pub fn new() -> Self { Self::default() }
-    pub fn push(&mut self, stats: StageStats) { self.stages.push(stats); }
-    pub fn stages(&self) -> &[StageStats] { &self.stages }
+    pub fn new() -> Self {
+        Self::default()
+    }
+    pub fn push(&mut self, stats: StageStats) {
+        self.stages.push(stats);
+    }
+    pub fn stages(&self) -> &[StageStats] {
+        &self.stages
+    }
     /// Print the summary to stderr.
-    pub fn print(&self) { eprint!("{self}"); }
+    pub fn print(&self) {
+        eprint!("{self}");
+    }
 }
 
 // ── diagnosis ─────────────────────────────────────────────────────────────────
 
 struct Diagnosis {
-    tag:    &'static str,
+    tag: &'static str,
     detail: Option<String>,
 }
 
 // Thresholds are intentionally conservative to avoid false positives.
 fn diagnose(s: &StageStats, n_cores: usize) -> Diagnosis {
-    let eff     = s.efficiency(n_cores);
+    let eff = s.efficiency(n_cores);
     let cpu_pct = eff * 100.0;
-    let io_ops  = s.in_blocks + s.out_blocks;
+    let io_ops = s.in_blocks + s.out_blocks;
 
     // swaps > 0 is the only reliable cross-platform indicator of true RAM exhaustion.
     // ru_majflt is intentionally excluded: on macOS it counts all file-backed mmap
@@ -387,26 +539,43 @@ fn diagnose(s: &StageStats, n_cores: usize) -> Diagnosis {
             )),
         };
     }
-    Diagnosis { tag: "—", detail: None }
+    Diagnosis {
+        tag: "—",
+        detail: None,
+    }
 }
 
 // ── display helpers ───────────────────────────────────────────────────────────
 
 fn fmt_secs(s: f64) -> String {
-    if s >= 100.0     { format!("{:.0}s",  s) }
-    else if s >= 10.0 { format!("{:.1}s",  s) }
-    else if s >= 1.0  { format!("{:.2}s",  s) }
-    else              { format!("{:.0}ms", s * 1000.0) }
+    if s >= 100.0 {
+        format!("{:.0}s", s)
+    } else if s >= 10.0 {
+        format!("{:.1}s", s)
+    } else if s >= 1.0 {
+        format!("{:.2}s", s)
+    } else {
+        format!("{:.0}ms", s * 1000.0)
+    }
 }
 
 fn fmt_bytes(b: u64) -> String {
-    if b >= 1 << 30      { format!("{:.1} GB", b as f64 / (1u64 << 30) as f64) }
-    else if b >= 1 << 20 { format!("{:.0} MB", b as f64 / (1u64 << 20) as f64) }
-    else                 { format!("{:.0} KB", b as f64 / 1024.0) }
+    if b >= 1 << 30 {
+        format!("{:.1} GB", b as f64 / (1u64 << 30) as f64)
+    } else if b >= 1 << 20 {
+        format!("{:.0} MB", b as f64 / (1u64 << 20) as f64)
+    } else {
+        format!("{:.0} KB", b as f64 / 1024.0)
+    }
 }
 
 fn fmt_efficiency(par: f64, n_cores: usize) -> String {
-    format!("{:.1}×/{} ({:.0}%)", par, n_cores, par / n_cores as f64 * 100.0)
+    format!(
+        "{:.1}×/{} ({:.0}%)",
+        par,
+        n_cores,
+        par / n_cores as f64 * 100.0
+    )
 }
 
 // ── Display ───────────────────────────────────────────────────────────────────
@@ -414,8 +583,8 @@ fn fmt_efficiency(par: f64, n_cores: usize) -> String {
 // ── MemoryBudget ──────────────────────────────────────────────────────────────
 
 struct BudgetInner {
-    remaining:   u64,
-    active:      usize,
+    remaining: u64,
+    active: usize,
     peak_active: usize,
 }
 
@@ -425,8 +594,8 @@ struct BudgetInner {
 /// completion. Non-deadlock guarantee: when no worker is active the next
 /// acquire always succeeds regardless of cost vs. remaining budget.
 pub struct MemoryBudget {
-    total:   u64,
-    inner:   Mutex<BudgetInner>,
+    total: u64,
+    inner: Mutex<BudgetInner>,
     condvar: Condvar,
 }
 
@@ -434,7 +603,11 @@ impl MemoryBudget {
     pub fn new(total: u64) -> Self {
         Self {
             total,
-            inner:   Mutex::new(BudgetInner { remaining: total, active: 0, peak_active: 0 }),
+            inner: Mutex::new(BudgetInner {
+                remaining: total,
+                active: 0,
+                peak_active: 0,
+            }),
             condvar: Condvar::new(),
         }
     }
@@ -443,9 +616,9 @@ impl MemoryBudget {
         let mut g = self.inner.lock().unwrap();
         loop {
             if g.active == 0 || g.remaining >= cost {
-                g.remaining     = g.remaining.saturating_sub(cost);
-                g.active       += 1;
-                g.peak_active   = g.peak_active.max(g.active);
+                g.remaining = g.remaining.saturating_sub(cost);
+                g.active += 1;
+                g.peak_active = g.peak_active.max(g.active);
                 return;
             }
             g = self.condvar.wait(g).unwrap();
@@ -455,47 +628,66 @@ impl MemoryBudget {
     pub fn release(&self, cost: u64) {
         let mut g = self.inner.lock().unwrap();
         g.remaining = (g.remaining + cost).min(self.total);
-        g.active   -= 1;
+        g.active -= 1;
         self.condvar.notify_all();
     }
 
-    pub fn total(&self)       -> u64   { self.total }
-    pub fn active(&self)      -> usize { self.inner.lock().unwrap().active }
-    pub fn remaining(&self)   -> u64   { self.inner.lock().unwrap().remaining }
-    pub fn peak_active(&self) -> usize { self.inner.lock().unwrap().peak_active }
+    pub fn total(&self) -> u64 {
+        self.total
+    }
+    pub fn active(&self) -> usize {
+        self.inner.lock().unwrap().active
+    }
+    pub fn remaining(&self) -> u64 {
+        self.inner.lock().unwrap().remaining
+    }
+    pub fn peak_active(&self) -> usize {
+        self.inner.lock().unwrap().peak_active
+    }
 }
 
 // ── Display ───────────────────────────────────────────────────────────────────
 
 impl fmt::Display for Reporter {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if self.stages.is_empty() { return Ok(()); }
+        if self.stages.is_empty() {
+            return Ok(());
+        }
 
         let n_cores = std::thread::available_parallelism()
             .map(|n| n.get())
             .unwrap_or(1);
 
         // column widths
-        let nw = self.stages.iter().map(|s| s.label.len()).max().unwrap_or(5).max(5);
+        let nw = self
+            .stages
+            .iter()
+            .map(|s| s.label.len())
+            .max()
+            .unwrap_or(5)
+            .max(5);
         // efficiency col: worst-case width for this run's n_cores value
         let ew = format!("{:.1}×/{} (100%)", 99.9f64, n_cores).len();
 
         let sep_w = nw + 2 + 7 + 2 + ew + 2 + 8 + 2 + 12;
-        let sep   = "─".repeat(sep_w);
+        let sep = "─".repeat(sep_w);
 
         // header
-        writeln!(f, "{:<nw$}  {:>7}  {:>ew$}  {:>8}  status",
-            "stage", "wall", "efficiency", "peak RSS")?;
+        writeln!(
+            f,
+            "{:<nw$}  {:>7}  {:>ew$}  {:>8}  status",
+            "stage", "wall", "efficiency", "peak RSS"
+        )?;
         writeln!(f, "{sep}")?;
 
         // compute all diagnoses up front (needed for both table and footnotes)
-        let diagnoses: Vec<Diagnosis> = self.stages.iter()
-            .map(|s| diagnose(s, n_cores))
-            .collect();
+        let diagnoses: Vec<Diagnosis> = self.stages.iter().map(|s| diagnose(s, n_cores)).collect();
 
         // per-stage rows
         for (s, d) in self.stages.iter().zip(diagnoses.iter()) {
-            writeln!(f, "{:<nw$}  {:>7}  {:>ew$}  {:>8}  {}",
+            writeln!(
+                f,
+                "{:<nw$}  {:>7}  {:>ew$}  {:>8}  {}",
                 s.label,
                 fmt_secs(s.wall_secs),
                 fmt_efficiency(s.parallelism(), n_cores),
@@ -505,14 +697,21 @@ impl fmt::Display for Reporter {
         }
 
         // totals
-        let tw   = self.stages.iter().map(|s| s.wall_secs).sum::<f64>();
-        let tu   = self.stages.iter().map(|s| s.user_secs).sum::<f64>();
-        let ts   = self.stages.iter().map(|s| s.sys_secs).sum::<f64>();
-        let trss = self.stages.iter().map(|s| s.max_rss_bytes).max().unwrap_or(0);
+        let tw = self.stages.iter().map(|s| s.wall_secs).sum::<f64>();
+        let tu = self.stages.iter().map(|s| s.user_secs).sum::<f64>();
+        let ts = self.stages.iter().map(|s| s.sys_secs).sum::<f64>();
+        let trss = self
+            .stages
+            .iter()
+            .map(|s| s.max_rss_bytes)
+            .max()
+            .unwrap_or(0);
         let tpar = if tw > 1e-9 { (tu + ts) / tw } else { 0.0 };
 
         writeln!(f, "{sep}")?;
-        writeln!(f, "{:<nw$}  {:>7}  {:>ew$}  {:>8}",
+        writeln!(
+            f,
+            "{:<nw$}  {:>7}  {:>ew$}  {:>8}",
             "TOTAL",
             fmt_secs(tw),
             fmt_efficiency(tpar, n_cores),
@@ -520,7 +719,9 @@ impl fmt::Display for Reporter {
         )?;
 
         // bottleneck footnotes (only if at least one anomaly detected)
-        let bottlenecks: Vec<(&str, &str)> = self.stages.iter()
+        let bottlenecks: Vec<(&str, &str)> = self
+            .stages
+            .iter()
             .zip(diagnoses.iter())
             .filter_map(|(s, d)| d.detail.as_deref().map(|det| (s.label.as_str(), det)))
             .collect();

src/obitaxonomy/Cargo.toml

@@ -0,0 +1,6 @@
+[package]
+name = "obitaxonomy"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]

src/obitaxonomy/src/error.rs

@@ -0,0 +1,38 @@
+use std::fmt;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum TaxError {
+    /// Stored value does not start with the `taxonomy:/` prefix.
+    MissingPrefix,
+    /// Stored path contains no segments after the prefix.
+    EmptyPath,
+    /// Query pattern contains no segments (after stripping anchors).
+    EmptyPattern,
+    /// A segment has an empty name (e.g. consecutive `/`).
+    EmptySegmentName,
+    /// A segment has a trailing `@` with no rank name.
+    EmptyRankName { segment: String },
+    /// A segment contains more than one `@`.
+    AmbiguousRank { segment: String },
+}
+
+impl fmt::Display for TaxError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            TaxError::MissingPrefix =>
+                write!(f, "taxonomy path must start with \"taxonomy:/\""),
+            TaxError::EmptyPath =>
+                write!(f, "taxonomy path has no segments"),
+            TaxError::EmptyPattern =>
+                write!(f, "taxonomy query pattern has no segments"),
+            TaxError::EmptySegmentName =>
+                write!(f, "segment has an empty name"),
+            TaxError::EmptyRankName { segment } =>
+                write!(f, "segment has '@' with no rank name: {segment:?}"),
+            TaxError::AmbiguousRank { segment } =>
+                write!(f, "segment contains more than one '@': {segment:?}"),
+        }
+    }
+}
+
+impl std::error::Error for TaxError {}

src/obitaxonomy/src/lib.rs

@@ -0,0 +1,11 @@
+mod error;
+mod segment;
+mod segment_pattern;
+mod path;
+mod pattern;
+
+pub use error::TaxError;
+pub use segment::TaxSegment;
+pub use segment_pattern::SegmentPattern;
+pub use path::{TaxPath, PREFIX};
+pub use pattern::TaxPattern;

src/obitaxonomy/src/path.rs

@@ -0,0 +1,82 @@
+use std::fmt;
+use std::str::FromStr;
+
+use crate::error::TaxError;
+use crate::segment::TaxSegment;
+
+/// The prefix that marks a metadata value as a taxonomy path.
+pub const PREFIX: &str = "taxonomy:/";
+
+/// A rooted, `/`-separated taxonomy path with optional per-segment rank annotations.
+///
+/// Stored form: `taxonomy:/seg1@rank1/seg2/seg3@rank3`
+/// The leading `taxonomy:/` is the discriminator; the remainder is one or more
+/// `/`-separated segments, each of the form `name` or `name@rank`.
+///
+/// `@` is reserved and may not appear in segment names or rank names.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TaxPath {
+    segments: Vec<TaxSegment>,
+}
+
+impl TaxPath {
+    pub fn parse(s: &str) -> Result<Self, TaxError> {
+        let tail = s.strip_prefix(PREFIX).ok_or(TaxError::MissingPrefix)?;
+        if tail.is_empty() {
+            return Err(TaxError::EmptyPath);
+        }
+        let segments = tail.split('/')
+            .map(TaxSegment::parse)
+            .collect::<Result<Vec<_>, _>>()?;
+        Ok(Self { segments })
+    }
+
+    /// True if `self` is an ancestor of — or equal to — `other`.
+    ///
+    /// Comparison is by segment name only; rank annotations are ignored.
+    /// `self` must be a prefix of `other` at segment granularity.
+    pub fn is_ancestor_of(&self, other: &TaxPath) -> bool {
+        self.segments.len() <= other.segments.len()
+            && self.segments.iter().zip(other.segments.iter())
+                .all(|(a, b)| a.name() == b.name())
+    }
+
+    /// Returns the name of the first segment whose rank equals `rank`, if any.
+    pub fn name_at_rank(&self, rank: &str) -> Option<&str> {
+        self.segments.iter()
+            .find(|s| s.rank() == Some(rank))
+            .map(|s| s.name())
+    }
+
+    /// True if any segment has the given rank.
+    pub fn has_rank(&self, rank: &str) -> bool {
+        self.segments.iter().any(|s| s.rank() == Some(rank))
+    }
+
+    /// True if the path contains a segment with both the given rank and name.
+    pub fn matches_rank(&self, rank: &str, name: &str) -> bool {
+        self.segments.iter().any(|s| s.rank() == Some(rank) && s.name() == name)
+    }
+
+    pub fn segments(&self) -> &[TaxSegment] { &self.segments }
+    pub fn depth(&self)    -> usize          { self.segments.len() }
+    pub fn is_empty(&self) -> bool           { self.segments.is_empty() }
+}
+
+impl fmt::Display for TaxPath {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", PREFIX)?;
+        let mut first = true;
+        for seg in &self.segments {
+            if !first { write!(f, "/")?; }
+            write!(f, "{seg}")?;
+            first = false;
+        }
+        Ok(())
+    }
+}
+
+impl FromStr for TaxPath {
+    type Err = TaxError;
+    fn from_str(s: &str) -> Result<Self, Self::Err> { Self::parse(s) }
+}

src/obitaxonomy/src/pattern.rs

@@ -0,0 +1,72 @@
+use crate::error::TaxError;
+use crate::path::TaxPath;
+use crate::segment::TaxSegment;
+use crate::segment_pattern::SegmentPattern;
+
+/// A query pattern for matching against stored `TaxPath` values.
+///
+/// Syntax:
+///
+/// | Form     | Semantics |
+/// |----------|-----------|
+/// | `A/B`    | A then B as a contiguous sub-path, anywhere in the value |
+/// | `/A/B`   | value starts with A then B (start-anchored) |
+/// | `A/B$`   | value ends with A then B (end-anchored) |
+/// | `/A/B$`  | value is exactly A then B (fully anchored) |
+/// | `A@x/B`  | A with rank `x`, followed by B with any rank |
+///
+/// A segment pattern without `@` matches any segment with that name regardless
+/// of its stored rank.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TaxPattern {
+    start_anchored: bool,
+    end_anchored:   bool,
+    segments:       Vec<SegmentPattern>,
+}
+
+impl TaxPattern {
+    pub fn parse(s: &str) -> Result<Self, TaxError> {
+        let s = s.trim();
+
+        let start_anchored = s.starts_with('/');
+        let s = if start_anchored { &s[1..] } else { s };
+
+        let end_anchored = s.ends_with('$');
+        let s = if end_anchored { &s[..s.len() - 1] } else { s };
+
+        if s.is_empty() {
+            return Err(TaxError::EmptyPattern);
+        }
+
+        let segments = s.split('/')
+            .map(SegmentPattern::parse)
+            .collect::<Result<Vec<_>, _>>()?;
+
+        Ok(Self { start_anchored, end_anchored, segments })
+    }
+
+    /// True if this pattern matches `path` according to the anchor flags.
+    ///
+    /// The pattern must match a contiguous run of segments in the path.
+    /// Start/end anchors restrict where that run may begin or end.
+    pub fn matches(&self, path: &TaxPath) -> bool {
+        let n = self.segments.len();
+        let m = path.depth();
+
+        if n > m { return false; }
+
+        let segs = path.segments();
+        match (self.start_anchored, self.end_anchored) {
+            (true,  true)  => n == m && self.window_matches(segs, 0),
+            (true,  false) => self.window_matches(segs, 0),
+            (false, true)  => self.window_matches(segs, m - n),
+            (false, false) => (0..=(m - n)).any(|i| self.window_matches(segs, i)),
+        }
+    }
+
+    fn window_matches(&self, segs: &[TaxSegment], start: usize) -> bool {
+        self.segments.iter()
+            .zip(segs[start..start + self.segments.len()].iter())
+            .all(|(pat, seg)| pat.matches(seg))
+    }
+}

src/obitaxonomy/src/segment.rs

@@ -0,0 +1,49 @@
+use std::fmt;
+
+use crate::error::TaxError;
+
+/// A single node in a taxonomy path: a name and an optional rank.
+///
+/// Neither `name` nor `rank` may contain `@` (reserved separator).
+/// Serialised form: `name` or `name@rank`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TaxSegment {
+    name: String,
+    rank: Option<String>,
+}
+
+impl TaxSegment {
+    pub fn parse(raw: &str) -> Result<Self, TaxError> {
+        let parts: Vec<&str> = raw.splitn(3, '@').collect();
+
+        let (name_raw, rank_raw) = match parts.as_slice() {
+            [name]        => (*name, None),
+            [name, rank]  => (*name, Some(*rank)),
+            _             => return Err(TaxError::AmbiguousRank { segment: raw.to_string() }),
+        };
+
+        if name_raw.is_empty() {
+            return Err(TaxError::EmptySegmentName);
+        }
+
+        let rank = match rank_raw {
+            None     => None,
+            Some("") => return Err(TaxError::EmptyRankName { segment: raw.to_string() }),
+            Some(r)  => Some(r.to_string()),
+        };
+
+        Ok(Self { name: name_raw.to_string(), rank })
+    }
+
+    pub fn name(&self) -> &str { &self.name }
+    pub fn rank(&self) -> Option<&str> { self.rank.as_deref() }
+}
+
+impl fmt::Display for TaxSegment {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match &self.rank {
+            None    => write!(f, "{}", self.name),
+            Some(r) => write!(f, "{}@{}", self.name, r),
+        }
+    }
+}

src/obitaxonomy/src/segment_pattern.rs

@@ -0,0 +1,41 @@
+use crate::error::TaxError;
+use crate::segment::TaxSegment;
+
+/// A single segment in a query pattern: a required name and an optional rank filter.
+///
+/// If `rank` is `None`, the pattern matches any segment with the given name,
+/// regardless of its stored rank. If `rank` is `Some(r)`, both name and rank
+/// must match exactly.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SegmentPattern {
+    name: String,
+    rank: Option<String>,
+}
+
+impl SegmentPattern {
+    pub fn parse(raw: &str) -> Result<Self, TaxError> {
+        let parts: Vec<&str> = raw.splitn(3, '@').collect();
+        let (name_raw, rank_raw) = match parts.as_slice() {
+            [name]       => (*name, None),
+            [name, rank] => (*name, Some(*rank)),
+            _            => return Err(TaxError::AmbiguousRank { segment: raw.to_string() }),
+        };
+        if name_raw.is_empty() {
+            return Err(TaxError::EmptySegmentName);
+        }
+        let rank = match rank_raw {
+            None     => None,
+            Some("") => return Err(TaxError::EmptyRankName { segment: raw.to_string() }),
+            Some(r)  => Some(r.to_string()),
+        };
+        Ok(Self { name: name_raw.to_string(), rank })
+    }
+
+    /// True if this pattern matches `seg`.
+    /// Name must match exactly. If a rank is specified in the pattern, the
+    /// segment's rank must match; otherwise any rank (or no rank) is accepted.
+    pub fn matches(&self, seg: &TaxSegment) -> bool {
+        self.name == seg.name()
+            && self.rank.as_deref().map_or(true, |r| seg.rank() == Some(r))
+    }
+}

test.sk.fasta

@@ -1,28 +0,0 @@
->F1FE4776BF3E1F06 {"seq_length":51,"kmer_size":31,"minimizer_size":11,"partition":229,"minimizer":"AAAAAAAATTA"}
-GAGTATACTCATGTGAGGGTAAAAAAAATTAAGTCCCATATTGAAACATTA
->C14BF81526DD6CB7 {"seq_length":31,"kmer_size":31,"minimizer_size":11,"partition":84,"minimizer":"AAAAAAATTAA"}
-AAAAAAATTAAGTCCCATATTGAAACATTAT
->9156D79605E4AC23 {"seq_length":31,"kmer_size":31,"minimizer_size":11,"partition":87,"minimizer":"AAAAAATTAAG"}
-AAAAAATTAAGTCCCATATTGAAACATTATC
->74666D1D78812D1E {"seq_length":31,"kmer_size":31,"minimizer_size":11,"partition":118,"minimizer":"AAAAATTAAGT"}
-AAAAATTAAGTCCCATATTGAAACATTATCA
->45EEFC3520FBDA9A {"seq_length":31,"kmer_size":31,"minimizer_size":11,"partition":32,"minimizer":"AAAATTAAGTC"}
-AAAATTAAGTCCCATATTGAAACATTATCAC
->5F44864B90170AF4 {"seq_length":49,"kmer_size":31,"minimizer_size":11,"partition":137,"minimizer":"AAACATTATCA"}
-AAATTAAGTCCCATATTGAAACATTATCACAAATGTGAGTTGTTAATAT
->8D10A11C86F8EF26 {"seq_length":42,"kmer_size":31,"minimizer_size":11,"partition":26,"minimizer":"AAATGTGAGTT"}
-AACATTATCACAAATGTGAGTTGTTAATATTACATAATTGGG
->C18F1086D0AF6E34 {"seq_length":32,"kmer_size":31,"minimizer_size":11,"partition":9,"minimizer":"TGTGAGTTGTT"}
-AATGTGAGTTGTTAATATTACATAATTGGGTT
->933477394DAF03BB {"seq_length":31,"kmer_size":31,"minimizer_size":11,"partition":48,"minimizer":"TAATTGGGTTT"}
-TGTGAGTTGTTAATATTACATAATTGGGTTT
->3CEE7E5227956042 {"seq_length":36,"kmer_size":31,"minimizer_size":11,"partition":252,"minimizer":"AATTGGGTTTT"}
-GTGAGTTGTTAATATTACATAATTGGGTTTTATGCT
->1BAF5B8767D63D0B {"seq_length":33,"kmer_size":31,"minimizer_size":11,"partition":201,"minimizer":"AAAGGCTCCCT"}
-TGAAAGGCTCCCTAGCGTGTTAATTAATCTCCC
->8368A897DB263C6F {"seq_length":38,"kmer_size":31,"minimizer_size":11,"partition":22,"minimizer":"CCTAGCGTGTT"}
-AAGGCTCCCTAGCGTGTTAATTAATCTCCCTGACAAGT
->247DC82E11CF8055 {"seq_length":35,"kmer_size":31,"minimizer_size":11,"partition":128,"minimizer":"AATCTCCCTGA"}
-CTAGCGTGTTAATTAATCTCCCTGACAAGTAGTGT
->11C93BBC8A5F6327 {"seq_length":35,"kmer_size":31,"minimizer_size":11,"partition":62,"minimizer":"CAAGTAGTGTT"}
-GTGTTAATTAATCTCCCTGACAAGTAGTGTTAGTG