From 76cbd3a88680104caeacefc3b256d4bc26fbf87c Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Thu, 20 Aug 2026 09:58:49 +0200 Subject: [PATCH] Centralize partition metadata access and add layer introspection APIs Replaced scattered direct metadata loading with centralized instance methods on `KmerPartition` to guarantee consistent error mapping and legacy recovery. Introduced `StorageKind`, `LayerContent`, and `EvidenceKind` enums alongside lightweight disk-probe methods that inspect file presence without opening heavy data structures. Updated callers across the index, partitioner, and phylo modules to use the new partition API, and added unit tests validating the introspection behavior. --- .../implementation/partition_layer_cache.md | 119 ++++++++++++++++- src/obicompactvec/src/bitmatrix/persistent.rs | 37 ++++++ src/obicompactvec/src/intmatrix.rs | 28 ++++ src/obicompactvec/src/lib.rs | 2 + src/obicompactvec/src/storage_kind.rs | 19 +++ src/obikindex/src/index.rs | 20 +-- src/obikindex/src/reindex.rs | 16 +-- src/obikindex/src/stats.rs | 9 +- src/obikpartitionner/src/dump_layer.rs | 9 +- src/obikpartitionner/src/merge_layer/mod.rs | 3 +- .../src/partition/kmer_partition.rs | 23 ++++ src/obikpartitionner/src/query_layer.rs | 3 +- src/obikpartitionner/src/select_layer.rs | 10 +- src/obikphylo/src/siblings/build.rs | 13 +- src/obikphylo/src/siblings/cache.rs | 6 +- src/obikphylo/src/siblings/family_scan.rs | 4 +- src/obikphylo/src/siblings/tests.rs | 14 +- src/obilayeredmap/src/layer.rs | 92 +++++++++++++ src/obilayeredmap/src/lib.rs | 4 +- src/obilayeredmap/src/map.rs | 31 ++++- src/obilayeredmap/src/mphf_layer.rs | 39 ++++++ src/obilayeredmap/src/tests/layer.rs | 30 +++++ src/obilayeredmap/src/tests/map.rs | 123 +++++++++++++++++- 23 files changed, 580 insertions(+), 74 deletions(-) create mode 100644 src/obicompactvec/src/storage_kind.rs diff --git a/DevDocMD/implementation/partition_layer_cache.md b/DevDocMD/implementation/partition_layer_cache.md index f3208fcd..e154be9b 100644 --- a/DevDocMD/implementation/partition_layer_cache.md +++ b/DevDocMD/implementation/partition_layer_cache.md @@ -1,7 +1,9 @@ # Partition and layer caching (discussion) -Status: problem confirmed, design direction agreed (2026-08-20). No -implementation started. Ownership split (below) still open. +Status: problem confirmed, design direction agreed (2026-08-20). (1)/(2) +themselves not started; ownership split (below) still open. Preparatory +encapsulation work (partition/layer path and metadata accessors on +`KmerPartition`) landed the same day — see "Preparatory work done" below. ## The problem @@ -127,3 +129,116 @@ deleted outright or kept as a thin sibling-specific wrapper, and whether (2) needs an eviction policy or can simply hold every partition open for the process lifetime (revisit once the VM-mapping-count question above has a real number behind it for this codebase's scale). + +## Preparatory work done (2026-08-20) + +Groundwork for (1)/(2), landed ahead of the design itself: + +- `KmerPartition` (`obikpartitionner`) gained `partition_dir`/`index_dir`/ + `layer_dir` as the single source of truth for a partition's on-disk + layout, replacing per-module duplicated `const INDEX_SUBDIR: &str = + "index"` (7 copies) and ad hoc path joins — including one found + duplicated *inside `KmerPartition` itself* (`ensure_writer` rebuilt + `part_dir`'s own logic by hand). +- `KmerPartition` gained `partition_meta`/`n_layers`/`index_mode`, wrapping + `obilayeredmap::meta::PartitionMeta::load` (via the existing + `common::load_meta`, which also recovers indexes built before + `meta.json` existed). Before this, `obikphylo` and `obikindex` imported + `obilayeredmap::meta::PartitionMeta` directly and called `::load()` + themselves at 21 call sites, each redoing its own error-mapping — + every one of those crates knew the on-disk metadata format instead of + going through an interface. Fixed everywhere except one remaining spot + (below). Caught as a side effect: `dump_layer.rs`/`query_layer.rs` had + been calling `PartitionMeta::load` directly, bypassing `load_meta` + entirely — they never got the missing-`meta.json` recovery the other + callers did. +- Layer introspection API discussed but **not yet implemented** — three + axes, deliberately kept separate after an initial draft conflated them: + - `LayerContent { Count, Presence }` — what the layer stores; a `const` + on `LayerData` (compile-time, zero-cost), not a runtime field. + - `StorageKind { Implicit, Columnar, Packed, Sparse }` — how it's + stored; only meaningful for `D` that actually carry data (`Layer<()>` + has neither this nor `LayerContent` — it's a write-time-only state, + never a queryable content: once a layer is closed, "no matrix file" + reads back as `Presence`/`Implicit` via `PersistentBitMatrix::open`'s + own fallback, not as some third "empty" content). + - `EvidenceKind { Exact, Approx, Hybrid }` — from `MphfLayer`'s own + already-in-memory `LayerEvidence` discriminant. + - Not all `(LayerContent, StorageKind)` pairs are legal: `Count` never + has `Implicit` or `Sparse`. + +**Implemented (2026-08-20).** `LayerContent`/`StorageKind`/`EvidenceKind` +now exist, each with two forms: +- A runtime accessor on an already-open value (`Layer::content()`/ + `storage_kind()`/`evidence_kind()`, `PersistentBitMatrix::storage_kind()`, + `PersistentCompactIntMatrix::storage_kind()`, `MphfLayer::evidence_kind()`) + — reads a discriminant already in memory, zero disk access. +- A lightweight `detect()`/`detect_storage()` disk probe that mirrors the + corresponding `open()`'s own priority order by hand (file-existence + checks only, no mmap) — usable *before* committing to a `D`, unlike the + runtime accessors. Exposed per-layer on `LayeredMap` as + `detect_layer_content`/`detect_layer_storage`/`detect_layer_evidence` + (work regardless of `D`, since they only use `self.root` + the layer + index). + +`StorageKind` lives in `obicompactvec` (owner of `PersistentBitMatrix`/ +`PersistentCompactIntMatrix`); `LayerContent`/`EvidenceKind` live in +`obilayeredmap`. `HasLayerContent`/`HasStorageKind` gate `Layer<()>` out of +`content()`/`storage_kind()` (no matrix, nothing to report), matching the +"empty is transitional" conclusion above. 42 new tests across +`obilayeredmap`'s `tests/layer.rs` and `tests/map.rs`; full workspace +suite green (0 failed) after. + +Not done: these `detect()` probes don't yet replace `Mat::open`'s or +`QueryLayer::open`'s own hand-rolled equivalents (still duplicated content/ +storage decisions, now a *third* copy of the same logic to keep in sync) +— that consolidation is (1)/(2)'s job, not this prep step's. + +## One bug found while reading around this (signalled, not fixed); one earlier claim retracted + +- `obicompactvec::bitmatrix::sparse.rs`'s module doc says + "Not used by any production code path yet" — false since `obikmer pack + --sparse` (`cmd/pack/mod.rs`) is wired to `pack_sparse_bit_matrix` and + `Mat::open` already reads the result back in the sibling-annex path. + Stale comment, not corrected. +- **Retracted (2026-08-20)**: an earlier pass through this doc claimed + `obikpartitionner::query_layer::QueryLayer::open` had no sparse-format + detection and would silently corrupt reads on a `pack --sparse`d layer. + False — `PersistentBitMatrix` (`obicompactvec::bitmatrix::persistent`) + is a 4-way enum (`Columnar`/`Packed`/`Sparse`/`Implicit`), not 3-way as + first read; its `open()` already detects `Sparse` via + `presence/sparse_meta.json`, and every method on the type (`row`, + `fill_row`, `nonzero_iter`, …) already dispatches all 4 arms. + `QueryLayer::open`'s `PersistentBitMatrix::open(layer_dir)` call was + never the bug. Root cause of the false claim: a `grep -n + "Implicit\|Columnar\|Packed"` used to read the enum definition silently + skipped the `Sparse(...)` line because it matched none of those three + words — a self-inflicted blind spot from a filtered read, not a fact + about the code. Lesson: for a `pub enum` whose variant list matters, + read the definition unfiltered, don't grep for the variant names you + expect to find. +- One real consequence of that same correction: `obikphylo::siblings:: + cache::Mat::SparsePresence(Layer)` looks + redundant now — `Mat::Presence(Layer)` alone + would already handle sparse layers transparently, since + `PersistentBitMatrix` absorbs `Sparse` internally. Likely `Mat` predates + `PersistentBitMatrix` growing native sparse support. Signalled, not + removed — no mandate to touch `obikphylo` for this. + +## Remaining instance of the PartitionMeta-encapsulation problem + +`obikphylo::siblings::family_scan::scan_layer_families` still re-derives +`index_dir` from `layer_dir.parent()` and calls `PartitionMeta::load` +itself, purely to get `.mode` for `Mat::open`. Fixing it the way the 21 +other call sites were fixed needs more than a 1:1 swap: `scan_layer_families` +only receives a bare `layer_dir: &Path`, not a `(partition, part, layer)` +triple, and its single upstream source of layer paths, +`sibling_layer_dirs`, returns a flat `Vec` with the partition/layer +indices already discarded. Fixing it properly means either having +`sibling_layer_dirs` return `(PathBuf, IndexMode)` (or `(part, layer)`) +pairs, or threading `&KmerPartition` + indices through instead of paths — +and touching every one of `scan_layer_families`'s 8 callers (`distance.rs`, +`alignment.rs`, `cardinality.rs`, `entropy.rs` ×2, `sankoff_bundle.rs` ×2, +`stats.rs`). Left alone this round; worth doing as part of the same pass +that builds (1)/(2), since those callers are exactly the sibling-annex +consumers (2) is meant to serve. diff --git a/src/obicompactvec/src/bitmatrix/persistent.rs b/src/obicompactvec/src/bitmatrix/persistent.rs index 71fa4113..ac94e033 100644 --- a/src/obicompactvec/src/bitmatrix/persistent.rs +++ b/src/obicompactvec/src/bitmatrix/persistent.rs @@ -6,6 +6,7 @@ use ndarray::{Array1, Array2}; use crate::bitvec::{PersistentBitVec, PersistentBitVecBuilder}; use crate::layer_meta::LayerMeta; use crate::meta::MatrixMeta; +use crate::storage_kind::StorageKind; use crate::traits::{BitPartials, ColumnWeights}; use crate::views::{nonzero_triples, BitSliceView}; @@ -63,6 +64,42 @@ impl PersistentBitMatrix { Ok(Self::Implicit { n_rows: meta.n, n_cols: 1 }) } + /// This already-open matrix's storage format — reads the discriminant + /// already in memory, no disk access. + #[inline] + pub fn storage_kind(&self) -> StorageKind { + match self { + Self::Columnar(_) => StorageKind::Columnar, + Self::Packed(_) => StorageKind::Packed, + Self::Sparse(_) => StorageKind::Sparse, + Self::Implicit { .. } => StorageKind::Implicit, + } + } + + /// Lightweight disk probe: which format `open` would pick, without + /// opening (mmap'ing) anything. Mirrors `open`'s own priority order by + /// hand — keep the two in sync if that order ever changes. + pub fn detect_storage(layer_dir: &Path) -> io::Result { + let presence_dir = layer_dir.join("presence"); + + if presence_dir.join("matrix.pbmx").exists() { + return Ok(StorageKind::Packed); + } + if MatrixMeta::load(&presence_dir).is_ok() { + return Ok(StorageKind::Columnar); + } + if presence_dir.join("sparse_meta.json").exists() { + return Ok(StorageKind::Sparse); + } + if LayerMeta::load(layer_dir).is_ok() { + return Ok(StorageKind::Implicit); + } + Err(io::Error::new( + io::ErrorKind::NotFound, + format!("cannot detect presence storage format in {}", layer_dir.display()), + )) + } + #[inline] pub fn n(&self) -> usize { match self { diff --git a/src/obicompactvec/src/intmatrix.rs b/src/obicompactvec/src/intmatrix.rs index dbb36338..2131e461 100644 --- a/src/obicompactvec/src/intmatrix.rs +++ b/src/obicompactvec/src/intmatrix.rs @@ -11,6 +11,7 @@ use crate::colgroup::{chunked_presence_count, ColGroup, MatrixGroupOps}; use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE}; use crate::meta::MatrixMeta; use crate::reader::PersistentCompactIntVec; +use crate::storage_kind::StorageKind; use crate::tempbitvec::{TempBitVec, TempBitVecBuilder}; use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder}; use crate::views::{nonzero_triples, IntSliceView}; @@ -287,6 +288,33 @@ impl PersistentCompactIntMatrix { )) } + /// This already-open matrix's storage format — reads the discriminant + /// already in memory, no disk access. + #[inline] + pub fn storage_kind(&self) -> StorageKind { + match self { + Self::Columnar(_) => StorageKind::Columnar, + Self::Packed(_) => StorageKind::Packed, + } + } + + /// Lightweight disk probe: which format `open` would pick, without + /// opening (mmap'ing) anything. Mirrors `open`'s own priority order by + /// hand — keep the two in sync if that order ever changes. + pub fn detect_storage(layer_dir: &Path) -> io::Result { + let counts_dir = layer_dir.join("counts"); + if counts_dir.join("matrix.pcmx").exists() { + return Ok(StorageKind::Packed); + } + if MatrixMeta::load(&counts_dir).is_ok() { + return Ok(StorageKind::Columnar); + } + Err(io::Error::new( + io::ErrorKind::NotFound, + format!("no count matrix found in {}", layer_dir.display()), + )) + } + #[inline] pub fn n(&self) -> usize { match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows } diff --git a/src/obicompactvec/src/lib.rs b/src/obicompactvec/src/lib.rs index a7ce5df3..caa08ef4 100644 --- a/src/obicompactvec/src/lib.rs +++ b/src/obicompactvec/src/lib.rs @@ -10,6 +10,7 @@ mod intmatrix; mod layer_meta; mod meta; mod reader; +mod storage_kind; mod tempbitvec; mod tempintvec; mod views; @@ -25,6 +26,7 @@ pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask}; pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix}; pub use layer_meta::LayerMeta; pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter}; +pub use storage_kind::StorageKind; pub use tempbitvec::{TempBitVec, TempBitVecBuilder}; pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder}; pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials}; diff --git a/src/obicompactvec/src/storage_kind.rs b/src/obicompactvec/src/storage_kind.rs new file mode 100644 index 00000000..10374a85 --- /dev/null +++ b/src/obicompactvec/src/storage_kind.rs @@ -0,0 +1,19 @@ +//! How a persistent matrix's data is physically laid out on disk — +//! orthogonal to *what* it stores (count vs. presence, `obilayeredmap`'s +//! `LayerContent` concern, one crate up). `PersistentCompactIntMatrix` only +//! ever reports `Columnar`/`Packed`; `PersistentBitMatrix` is the only type +//! that can also report `Sparse`/`Implicit`. + +/// On-disk storage format of a persistent matrix. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StorageKind { + /// Per-column files (`.pbiv`/`.pciv`), the build-time format. + Columnar, + /// Single mmap'd file (`matrix.pbmx`/`matrix.pcmx`), query-optimised. + Packed, + /// Row-major deduplicated sparse format (`PersistentBitMatrix` only). + Sparse, + /// No file at all — mono-genome, all values present + /// (`PersistentBitMatrix` only). + Implicit, +} diff --git a/src/obikindex/src/index.rs b/src/obikindex/src/index.rs index d178113f..355483d0 100644 --- a/src/obikindex/src/index.rs +++ b/src/obikindex/src/index.rs @@ -3,7 +3,6 @@ use std::fs; use std::path::{Path, PathBuf}; use obikpartitionner::{KmerPartition, KmerSpectrum, PARTITIONS_SUBDIR}; -use obilayeredmap; use obisys::{Reporter, Stage, progress_bar}; use rayon::prelude::*; use tracing::info; @@ -148,11 +147,7 @@ impl KmerIndex { /// homogeneous across all partitions — reading it off partition 0 /// is enough, no need to scan every partition. pub fn n_layers_per_partition(&self) -> OKIResult { - use obilayeredmap::meta::PartitionMeta; - let index_dir = self.partition.index_dir(0); - let meta = PartitionMeta::load(&index_dir) - .map_err(|e| OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))?; - Ok(meta.n_layers) + Ok(self.partition.n_layers(0)?) } /// Expose the inner partition so the caller can run scatter into it. @@ -279,7 +274,6 @@ impl KmerIndex { /// (see the sparse-matrix design plan's "Explicitly deferred"). pub fn pack_matrices(&self, sparse: bool) -> OKIResult<()> { use obicompactvec::{pack_bit_matrix, pack_compact_int_matrix, pack_sparse_bit_matrix}; - use obilayeredmap::meta::PartitionMeta; let n = self.n_partitions(); let order: Vec = (0..n).collect(); @@ -289,9 +283,8 @@ impl KmerIndex { |i| -> OKIResult<()> { let index_dir = self.partition.index_dir(i); if !index_dir.exists() { return Ok(()); } - let meta = PartitionMeta::load(&index_dir) - .map_err(|e| OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))?; - for l in 0..meta.n_layers { + let n_layers = self.partition.n_layers(i)?; + for l in 0..n_layers { let layer_dir = self.partition.layer_dir(i, l); let presence_dir = layer_dir.join("presence"); let counts_dir = layer_dir.join("counts"); @@ -319,7 +312,6 @@ impl KmerIndex { pub fn upgrade_layer_meta(&self) -> OKIResult<()> { use obicompactvec::LayerMeta; use obiskio::UnitigFileReader; - use obilayeredmap::meta::PartitionMeta; let n = self.n_partitions(); let errors: Vec<_> = (0..n) @@ -327,11 +319,11 @@ impl KmerIndex { .filter_map(|i| { let index_dir = self.partition.index_dir(i); if !index_dir.exists() { return None; } - let meta = match PartitionMeta::load(&index_dir) { - Ok(m) => m, + let n_layers = match self.partition.n_layers(i) { + Ok(n) => n, Err(e) => return Some(OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))), }; - for l in 0..meta.n_layers { + for l in 0..n_layers { let layer_dir = self.partition.layer_dir(i, l); let meta_path = layer_dir.join(LayerMeta::FILENAME); if meta_path.exists() { continue; } diff --git a/src/obikindex/src/reindex.rs b/src/obikindex/src/reindex.rs index 05a919e1..932fee23 100644 --- a/src/obikindex/src/reindex.rs +++ b/src/obikindex/src/reindex.rs @@ -1,7 +1,7 @@ use std::fs; use std::path::Path; -use obilayeredmap::{layer_dir, IndexMode, layer::Layer}; -use obilayeredmap::meta::PartitionMeta; +use obikpartitionner::KmerPartition; +use obilayeredmap::{IndexMode, layer::Layer}; use obisys::{Reporter, Stage, progress_bar}; use tracing::info; @@ -48,7 +48,7 @@ impl KmerIndex { let runner = crate::numa::PartitionRunner::new(); runner.run( &order, - |i| reindex_partition(&self.partition.index_dir(i), &target, block_bits) + |i| reindex_partition(&self.partition, i, &target, block_bits) .map_err(|e| OKIError::InvalidInput(format!("partition {i}: {e}"))), |_, _, _| { pb.inc(1); }, )?; @@ -66,13 +66,13 @@ impl KmerIndex { } /// Process all layers of one partition's index directory. -fn reindex_partition(index_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> { - if !index_dir.exists() { +fn reindex_partition(partition: &KmerPartition, i: usize, target: &IndexMode, block_bits: u8) -> OKIResult<()> { + if !partition.index_dir(i).exists() { return Ok(()); } - let pm = PartitionMeta::load(index_dir).map_err(olm_to_oki)?; - for layer_idx in 0..pm.n_layers { - reindex_layer(&layer_dir(index_dir, layer_idx), target, block_bits)?; + let n_layers = partition.n_layers(i).map_err(|e| OKIError::InvalidInput(e.to_string()))?; + for layer_idx in 0..n_layers { + reindex_layer(&partition.layer_dir(i, layer_idx), target, block_bits)?; } Ok(()) } diff --git a/src/obikindex/src/stats.rs b/src/obikindex/src/stats.rs index eef2a252..995b7c20 100644 --- a/src/obikindex/src/stats.rs +++ b/src/obikindex/src/stats.rs @@ -3,7 +3,6 @@ use std::path::Path; use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix}; use obicompactvec::traits::ColumnWeights; -use obilayeredmap::meta::PartitionMeta; use rayon::prelude::*; use crate::error::OKIResult; @@ -93,9 +92,7 @@ impl KmerIndex { let index_dir = self.partition.index_dir(i); if !index_dir.exists() { return (0usize, 0u64, 0u64, 0u64); } - let n_layers = PartitionMeta::load(&index_dir) - .map(|m| m.n_layers) - .unwrap_or(0); + let n_layers = self.partition.n_layers(i).unwrap_or(0); (0..n_layers).fold((0usize, 0u64, 0u64, 0u64), |acc, l| { let lb = layer_bytes(&self.partition.layer_dir(i, l)); @@ -144,9 +141,7 @@ impl KmerIndex { let index_dir = self.partition.index_dir(i); if !index_dir.exists() { return (0, counts); } - let n_layers = PartitionMeta::load(&index_dir) - .map(|m| m.n_layers) - .unwrap_or(0); + let n_layers = self.partition.n_layers(i).unwrap_or(0); for l in 0..n_layers { let this_layer_dir = self.partition.layer_dir(i, l); diff --git a/src/obikpartitionner/src/dump_layer.rs b/src/obikpartitionner/src/dump_layer.rs index 46e93cda..eb18af0e 100644 --- a/src/obikpartitionner/src/dump_layer.rs +++ b/src/obikpartitionner/src/dump_layer.rs @@ -2,7 +2,6 @@ use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obikseq::CanonicalKmer; use obiskio::{SKError, SKResult, UnitigFileReader}; use obilayeredmap::{IndexMode, MphfLayer, OLMError}; -use obilayeredmap::meta::PartitionMeta; use crate::filter::{KmerFilter, passes_all}; use crate::partition::KmerPartition; @@ -39,9 +38,7 @@ impl KmerPartition { return Ok(true); } - let index_mode = PartitionMeta::load(&index_dir) - .map(|m| m.mode) - .unwrap_or(IndexMode::Exact); + let index_mode = self.index_mode(part).unwrap_or(IndexMode::Exact); let mut l = 0; loop { @@ -119,9 +116,7 @@ impl KmerPartition { return Ok(true); } - let index_mode = PartitionMeta::load(&index_dir) - .map(|m| m.mode) - .unwrap_or(IndexMode::Exact); + let index_mode = self.index_mode(part).unwrap_or(IndexMode::Exact); let mut layer = 0; loop { diff --git a/src/obikpartitionner/src/merge_layer/mod.rs b/src/obikpartitionner/src/merge_layer/mod.rs index ea92e3f1..1fc68811 100644 --- a/src/obikpartitionner/src/merge_layer/mod.rs +++ b/src/obikpartitionner/src/merge_layer/mod.rs @@ -21,7 +21,6 @@ use obipipeline::{ use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder}; use obikseq::CanonicalKmer; -use obilayeredmap::meta::PartitionMeta; use obilayeredmap::{layer_dir, IndexMode, Layer, LayeredMap, MphfOnly}; use obiskio::{SKError, SKResult, UnitigFileReader}; @@ -526,7 +525,7 @@ impl KmerPartition { if let Some(mb) = new_mb { mb.close().map_err(SKError::Io)?; - let mut part_meta = PartitionMeta::load(&dst_index_dir).map_err(|e| olm_to_sk(e, "merge"))?; + let mut part_meta = self.partition_meta(i)?; part_meta.n_layers = new_layer_idx + 1; part_meta.save(&dst_index_dir).map_err(|e| olm_to_sk(e, "merge"))?; } diff --git a/src/obikpartitionner/src/partition/kmer_partition.rs b/src/obikpartitionner/src/partition/kmer_partition.rs index ec3a5fa4..57a202c2 100644 --- a/src/obikpartitionner/src/partition/kmer_partition.rs +++ b/src/obikpartitionner/src/partition/kmer_partition.rs @@ -8,6 +8,8 @@ use obisys::progress_bar; use obikseq::RoutableSuperKmer; use obiskio::SKResult; +use obilayeredmap::IndexMode; +use obilayeredmap::meta::PartitionMeta; use rayon::prelude::*; use remove_dir_all::remove_dir_all; use sysinfo::System; @@ -16,6 +18,7 @@ use niffler::Level; use niffler::send::compression::Format; use obiskio::SKFileWriter; +use crate::common::load_meta; use crate::kmer_sort::chunk_size_from_ram; use super::count::count_partition; @@ -191,6 +194,26 @@ impl KmerPartition { obilayeredmap::layer_dir(&self.index_dir(i), l) } + /// Partition `i`'s metadata (layer count, evidence mode) — the single + /// entry point for this, so that callers outside `obikpartitionner` + /// never need to know it's a `meta.json` loaded via + /// `obilayeredmap::meta::PartitionMeta`, nor handle its own recovery + /// path for indexes built before that file existed (see + /// [`crate::common::load_meta`]). + pub fn partition_meta(&self, i: usize) -> SKResult { + load_meta(&self.index_dir(i), "partition_meta") + } + + /// Number of layers in partition `i` — see [`partition_meta`](Self::partition_meta). + pub fn n_layers(&self, i: usize) -> SKResult { + self.partition_meta(i).map(|m| m.n_layers) + } + + /// Evidence mode of partition `i` — see [`partition_meta`](Self::partition_meta). + pub fn index_mode(&self, i: usize) -> SKResult { + self.partition_meta(i).map(|m| m.mode) + } + pub fn kmer_size(&self) -> usize { self.kmer_size } diff --git a/src/obikpartitionner/src/query_layer.rs b/src/obikpartitionner/src/query_layer.rs index 3c8cbc6a..fdd0ac84 100644 --- a/src/obikpartitionner/src/query_layer.rs +++ b/src/obikpartitionner/src/query_layer.rs @@ -5,7 +5,6 @@ use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obikseq::CanonicalKmer; use obiskio::{SKError, SKResult}; use obilayeredmap::{IndexMode, MphfLayer, OLMError}; -use obilayeredmap::meta::PartitionMeta; use crate::partition::KmerPartition; @@ -178,7 +177,7 @@ impl KmerPartition { return Ok(stats); } - let meta = PartitionMeta::load(&index_dir).map_err(olm_to_sk)?; + let meta = self.partition_meta(part_idx)?; let layers: Vec = (0..meta.n_layers) .map(|i| QueryLayer::open(&self.layer_dir(part_idx, i), with_counts, &meta.mode)) .collect::>()?; diff --git a/src/obikpartitionner/src/select_layer.rs b/src/obikpartitionner/src/select_layer.rs index 691dba55..158ab93c 100644 --- a/src/obikpartitionner/src/select_layer.rs +++ b/src/obikpartitionner/src/select_layer.rs @@ -7,7 +7,6 @@ use obicompactvec::{ PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, }; -use obilayeredmap::meta::PartitionMeta; use obilayeredmap::OLMError; use obiskio::{SKError, SKResult}; @@ -150,8 +149,8 @@ impl KmerPartition { return Ok(()); } - let src_meta = PartitionMeta::load(&src_index_dir).map_err(olm_to_sk)?; - if src_meta.n_layers == 0 { + let n_src_layers = src.n_layers(i)?; + if n_src_layers == 0 { return Ok(()); } @@ -162,7 +161,7 @@ impl KmerPartition { let data_subdir = if output_presence { "presence" } else { "counts" }; - for l in 0..src_meta.n_layers { + for l in 0..n_src_layers { let src_layer_dir = src.layer_dir(i, l); if !src_layer_dir.exists() { continue; } @@ -234,8 +233,7 @@ impl KmerPartition { } if !in_place { - PartitionMeta::load(&src_index_dir).map_err(olm_to_sk)? - .save(&dst_index_dir).map_err(olm_to_sk)?; + src.partition_meta(i)?.save(&dst_index_dir).map_err(olm_to_sk)?; } Ok(()) diff --git a/src/obikphylo/src/siblings/build.rs b/src/obikphylo/src/siblings/build.rs index 28c0768b..358b0778 100644 --- a/src/obikphylo/src/siblings/build.rs +++ b/src/obikphylo/src/siblings/build.rs @@ -8,7 +8,7 @@ use obikpartitionner::KmerPartition; use obipipeline::ThrottleGuard; use obikseq::CanonicalKmer; use obilayeredmap::MphfLayer; -use obilayeredmap::meta::PartitionMeta; +use obilayeredmap::meta::IndexMode; use obisys::progress_bar; use obikindex::{OKIError, OKIResult}; @@ -97,11 +97,13 @@ impl SiblingAnnexBuildExt for KmerIndex { pb.inc(1); continue; } - let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?; + let meta = self.partition().partition_meta(part)?; let mut part_slots: u64 = 0; for l in 0..meta.n_layers { - part_slots += build_layer_sibling_annex(self, &self.partition().layer_dir(part, l), n_parts, l, &cache)?; + part_slots += build_layer_sibling_annex( + self, &self.partition().layer_dir(part, l), &meta.mode, n_parts, l, &cache, + )?; } total_slots += part_slots; pb.inc(1); @@ -120,13 +122,12 @@ impl SiblingAnnexBuildExt for KmerIndex { fn build_layer_sibling_annex( index: &KmerIndex, layer_dir: &Path, + mode: &IndexMode, n_parts: usize, l: usize, cache: &Arc, ) -> OKIResult { - let index_dir = layer_dir.parent().expect("layer_dir has a parent index dir"); - let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?; - let mphf = MphfLayer::open(layer_dir, &meta.mode).map_err(olm_to_ok)?; + let mphf = MphfLayer::open(layer_dir, mode).map_err(olm_to_ok)?; let k = index.kmer_size(); let n = mphf.n(); diff --git a/src/obikphylo/src/siblings/cache.rs b/src/obikphylo/src/siblings/cache.rs index 8ffef9fe..315c0660 100644 --- a/src/obikphylo/src/siblings/cache.rs +++ b/src/obikphylo/src/siblings/cache.rs @@ -6,13 +6,13 @@ use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, PersistentS use obikpartitionner::KmerPartition; use obikseq::CanonicalKmer; use obilayeredmap::{Layer, OLMResult}; -use obilayeredmap::meta::{IndexMode, PartitionMeta}; +use obilayeredmap::meta::IndexMode; use obisys::progress_bar; use obikindex::OKIResult; use super::iter::SiblingLayerExt; -use super::{olm_to_ok, SiblingAnnex}; +use super::SiblingAnnex; /// Every partition's already-open layers, built **once** for the whole /// `build_sibling_annex` run and shared (read-only) across every lookup, in @@ -179,7 +179,7 @@ impl PartitionCache { pb.inc(1); return Ok((Vec::new(), 0)); } - let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?; + let meta = partition.partition_meta(part)?; let mut mats = Vec::with_capacity(meta.n_layers); for l in 0..meta.n_layers { let Ok(mat) = Mat::open(&partition.layer_dir(part, l), &meta.mode, with_counts) else { continue }; diff --git a/src/obikphylo/src/siblings/family_scan.rs b/src/obikphylo/src/siblings/family_scan.rs index 0215bc68..f36c3f73 100644 --- a/src/obikphylo/src/siblings/family_scan.rs +++ b/src/obikphylo/src/siblings/family_scan.rs @@ -109,8 +109,8 @@ pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult> { if !index_dir.exists() { continue; } - let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?; - for l in 0..meta.n_layers { + let n_layers = index.partition().n_layers(part)?; + for l in 0..n_layers { let this_layer_dir = index.partition().layer_dir(part, l); let annex_path = this_layer_dir.join(ANNEX_FILE_NAME); if !annex_path.exists() { diff --git a/src/obikphylo/src/siblings/tests.rs b/src/obikphylo/src/siblings/tests.rs index 93784782..929f49bf 100644 --- a/src/obikphylo/src/siblings/tests.rs +++ b/src/obikphylo/src/siblings/tests.rs @@ -3,7 +3,6 @@ use std::path::Path; use obikseq::{CanonicalKmer, Kmer, Sequence}; use obilayeredmap::MphfLayer; -use obilayeredmap::meta::PartitionMeta; use obisys::Reporter; use tempfile::tempdir; @@ -88,8 +87,7 @@ fn canonical(ascii: &[u8]) -> CanonicalKmer { /// Read back the annex entry for a given canonical k-mer from the merged /// index's (single) partition/layer, asserting it was found at all. fn annex_info_for(idx: &KmerIndex, kmer: CanonicalKmer) -> FamilyMask { - let index_dir = idx.partition().index_dir(0); - let meta = PartitionMeta::load(&index_dir).unwrap(); + let meta = idx.partition().partition_meta(0).unwrap(); for l in 0..meta.n_layers { let layer_dir = idx.partition().layer_dir(0, l); let mphf = MphfLayer::open(&layer_dir, &meta.mode).unwrap(); @@ -310,9 +308,8 @@ fn sibling_annex_no_empty_masks_after_build() { let g1 = build_single_genome_index(dir.path(), "g1", &seq); g1.build_sibling_annex().expect("build_sibling_annex"); - let index_dir = g1.partition().index_dir(0); - let meta = PartitionMeta::load(&index_dir).expect("partition meta"); - for l in 0..meta.n_layers { + let n_layers = g1.partition().n_layers(0).expect("partition meta"); + for l in 0..n_layers { let layer_dir = g1.partition().layer_dir(0, l); let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).expect("annex open"); for slot in 0..annex.len() { @@ -361,9 +358,8 @@ fn sibling_annex_records_the_real_layer_of_each_family_member() { let merged = merge_two(dir.path(), &g1, &g2); merged.build_sibling_annex().expect("build_sibling_annex"); - let index_dir = merged.partition().index_dir(0); - let meta = PartitionMeta::load(&index_dir).unwrap(); - assert_eq!(meta.n_layers, 2, "fixture assumption: one merge, one new layer"); + let n_layers = merged.partition().n_layers(0).unwrap(); + assert_eq!(n_layers, 2, "fixture assumption: one merge, one new layer"); let g1_kmer = canonical(b"AACCGCTTAAG"); // own base C (1), sibling base G (2) — lives in layer 1 let g2_kmer = canonical(b"AACCGGTTAAG"); // own base G (2), sibling base C (1) — lives in layer 0 diff --git a/src/obilayeredmap/src/layer.rs b/src/obilayeredmap/src/layer.rs index f0b22caa..44859e18 100644 --- a/src/obilayeredmap/src/layer.rs +++ b/src/obilayeredmap/src/layer.rs @@ -85,6 +85,76 @@ impl LayerData for PersistentSparseBitMatrix { fn read(&self, slot: usize) -> Box<[bool]> { self.row(slot) } } +// ── LayerContent ───────────────────────────────────────────────────────────── + +/// What a layer's data matrix represents — orthogonal to *how* it's stored +/// on disk (`obicompactvec::StorageKind`'s concern). Only meaningful for +/// `D` that actually carry a matrix: `Layer<()>` (mode 1, set membership, +/// no matrix at all) has neither a `LayerContent` nor a `content()` method +/// — it's a write-time-only state, never a queryable content. Once a layer +/// is closed, "no matrix file" reads back as `Presence` via +/// `PersistentBitMatrix::open`'s own `Implicit` fallback, not as some third +/// "empty" content — see `DevDocMD/implementation/partition_layer_cache.md`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LayerContent { + Count, + Presence, +} + +impl LayerContent { + /// Lightweight disk probe, no MPHF/matrix opened: `counts/` present → + /// `Count`, otherwise `Presence` — an absent/`Implicit` presence + /// matrix still reads as `Presence`, never a separate content. + pub fn detect(layer_dir: &Path) -> LayerContent { + if layer_dir.join(COUNTS_DIR).exists() { + LayerContent::Count + } else { + LayerContent::Presence + } + } +} + +/// Implemented only by `D` that carry a real matrix — gates +/// [`Layer::content`](Layer::content) so `Layer<()>` doesn't get it. +pub trait HasLayerContent { + const CONTENT: LayerContent; +} + +impl HasLayerContent for PersistentCompactIntMatrix { + const CONTENT: LayerContent = LayerContent::Count; +} + +impl HasLayerContent for PersistentBitMatrix { + const CONTENT: LayerContent = LayerContent::Presence; +} + +impl HasLayerContent for PersistentSparseBitMatrix { + const CONTENT: LayerContent = LayerContent::Presence; +} + +// ── StorageKind passthrough ──────────────────────────────────────────────────── + +/// Implemented only by `D` that expose their own `storage_kind()` — gates +/// [`Layer::storage_kind`]. `PersistentSparseBitMatrix` has a single, +/// fixed on-disk format (no `Columnar`/`Packed`/`Implicit` variants of its +/// own), so it's deliberately not given an impl: there's nothing to report +/// beyond what `content()` already says. +pub trait HasStorageKind { + fn storage_kind(&self) -> obicompactvec::StorageKind; +} + +impl HasStorageKind for PersistentCompactIntMatrix { + fn storage_kind(&self) -> obicompactvec::StorageKind { + PersistentCompactIntMatrix::storage_kind(self) + } +} + +impl HasStorageKind for PersistentBitMatrix { + fn storage_kind(&self) -> obicompactvec::StorageKind { + PersistentBitMatrix::storage_kind(self) + } +} + // ── Structures ──────────────────────────────────────────────────────────────── pub struct Layer { @@ -171,6 +241,28 @@ impl Layer { MphfLayer::build_approx_evidence(layer_dir, b, z) } + /// This already-open layer's evidence mode — reads the discriminant + /// already in memory (`MphfLayer`'s own `LayerEvidence`), no disk + /// access. Available regardless of `D`, unlike `content`/`storage_kind`. + pub fn evidence_kind(&self) -> crate::mphf_layer::EvidenceKind { + self.mphf.evidence_kind() + } +} + +impl Layer { + /// This already-open layer's content (`Count`/`Presence`) — a + /// compile-time fact about `D`, not a runtime read. + pub const fn content(&self) -> LayerContent { + D::CONTENT + } +} + +impl Layer { + /// This already-open layer's storage format — delegates to `D`'s own + /// `storage_kind()`, reading a discriminant already in memory. + pub fn storage_kind(&self) -> obicompactvec::StorageKind { + self.data.storage_kind() + } } // ── Mode 1 — set membership ─────────────────────────────────────────────────── diff --git a/src/obilayeredmap/src/lib.rs b/src/obilayeredmap/src/lib.rs index 103035eb..66387b27 100644 --- a/src/obilayeredmap/src/lib.rs +++ b/src/obilayeredmap/src/lib.rs @@ -8,8 +8,8 @@ pub mod meta; pub(crate) mod mphf_layer; pub use error::{OLMError, OLMResult}; -pub use layer::{layer_dir, open_data, Hit, Layer, LayerData}; +pub use layer::{layer_dir, open_data, HasLayerContent, HasStorageKind, Hit, Layer, LayerContent, LayerData}; pub use layered_store::LayeredStore; pub use map::LayeredMap; pub use meta::{IndexMode, PartitionMeta}; -pub use mphf_layer::{KmerBatchIter, KmerIter, MphfLayer, MphfOnly}; +pub use mphf_layer::{EvidenceKind, KmerBatchIter, KmerIter, MphfLayer, MphfOnly}; diff --git a/src/obilayeredmap/src/map.rs b/src/obilayeredmap/src/map.rs index 9a8971de..96d27975 100644 --- a/src/obilayeredmap/src/map.rs +++ b/src/obilayeredmap/src/map.rs @@ -2,13 +2,14 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; -use obicompactvec::PersistentCompactIntMatrix; +use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind}; use obikseq::CanonicalKmer; use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS}; -use crate::error::OLMResult; -use crate::layer::{layer_dir, Hit, Layer, LayerData}; +use crate::error::{OLMError, OLMResult}; +use crate::layer::{layer_dir, Hit, Layer, LayerContent, LayerData}; use crate::meta::{IndexMode, PartitionMeta}; +use crate::mphf_layer::EvidenceKind; /// Layered kmer index for a single partition. /// @@ -46,6 +47,30 @@ impl LayeredMap { pub fn layer(&self, i: usize) -> &Layer { &self.layers[i] } pub fn mode(&self) -> &IndexMode { &self.meta.mode } + /// Layer `i`'s content (`Count`/`Presence`) — a lightweight disk probe, + /// no MPHF/matrix opened. For picking a `D` before committing to + /// [`Layer::open`], or for reporting/diagnostics over an already-open + /// `LayeredMap` without re-opening every layer under a different `D`. + pub fn detect_layer_content(&self, i: usize) -> LayerContent { + LayerContent::detect(&layer_dir(&self.root, i)) + } + + /// Layer `i`'s storage format — a lightweight disk probe, no + /// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content). + pub fn detect_layer_storage(&self, i: usize) -> OLMResult { + let dir = layer_dir(&self.root, i); + match LayerContent::detect(&dir) { + LayerContent::Count => PersistentCompactIntMatrix::detect_storage(&dir).map_err(OLMError::Io), + LayerContent::Presence => PersistentBitMatrix::detect_storage(&dir).map_err(OLMError::Io), + } + } + + /// Layer `i`'s evidence mode — a lightweight disk probe, no + /// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content). + pub fn detect_layer_evidence(&self, i: usize) -> OLMResult { + EvidenceKind::detect(&layer_dir(&self.root, i)) + } + /// Query `kmer` across all layers. Returns `(layer_index, Hit)` on match. pub fn query(&self, kmer: CanonicalKmer) -> Option<(usize, Hit)> { self.layers diff --git a/src/obilayeredmap/src/mphf_layer.rs b/src/obilayeredmap/src/mphf_layer.rs index 6f47dc25..53588151 100644 --- a/src/obilayeredmap/src/mphf_layer.rs +++ b/src/obilayeredmap/src/mphf_layer.rs @@ -35,6 +35,35 @@ enum LayerEvidence { Hybrid { evidence: Evidence, unitigs: Arc, fingerprint: FingerprintVec }, } +// ── EvidenceKind ────────────────────────────────────────────────────────────── + +/// Coarse evidence mode of a layer — the `IndexMode` a layer was opened +/// with, without the `Approx`/`Hybrid` `b`/`z` parameters. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EvidenceKind { + Exact, + Approx, + Hybrid, +} + +impl EvidenceKind { + /// Lightweight disk probe, no MPHF/evidence opened: presence of + /// `evidence.bin`/`fingerprint.bin` alone determines the mode — same + /// signal `MphfLayer::open` uses, just without opening either file. + pub fn detect(layer_dir: &Path) -> OLMResult { + let has_evidence = layer_dir.join(EVIDENCE_FILE).exists(); + let has_fingerprint = layer_dir.join(FINGERPRINT_FILE).exists(); + match (has_evidence, has_fingerprint) { + (true, false) => Ok(EvidenceKind::Exact), + (false, true) => Ok(EvidenceKind::Approx), + (true, true) => Ok(EvidenceKind::Hybrid), + (false, false) => Err(OLMError::InvalidLayer(format!( + "no evidence.bin or fingerprint.bin in {}", layer_dir.display() + ))), + } + } +} + // ── MphfLayer ───────────────────────────────────────────────────────────────── /// Autonomous kmer → slot mapping for one layer. @@ -163,6 +192,16 @@ impl MphfLayer { pub fn n(&self) -> usize { self.n } + /// This already-open layer's evidence mode — reads the discriminant + /// already in memory, no disk access. + pub fn evidence_kind(&self) -> EvidenceKind { + match &self.ev { + LayerEvidence::Exact { .. } => EvidenceKind::Exact, + LayerEvidence::Approx { .. } => EvidenceKind::Approx, + LayerEvidence::Hybrid { .. } => EvidenceKind::Hybrid, + } + } + /// Raw MPHF lookup: kmer → slot. /// /// Returns the slot assigned by the MPHF without performing any membership diff --git a/src/obilayeredmap/src/tests/layer.rs b/src/obilayeredmap/src/tests/layer.rs index 81eed976..31a3bdb7 100644 --- a/src/obilayeredmap/src/tests/layer.rs +++ b/src/obilayeredmap/src/tests/layer.rs @@ -98,6 +98,36 @@ fn presence_layer_generic_over_sparse_matches_dense() { } } +// ── content / storage_kind / evidence_kind (runtime introspection) ───────── + +#[test] +fn count_layer_reports_count_content_and_columnar_storage() { + set_k(4); + let dir = tempdir().unwrap(); + write_unitigs(dir.path(), &[b"AAAACGT"]); + Layer::::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1) + .unwrap(); + let layer = Layer::::open(dir.path(), &IndexMode::Exact).unwrap(); + + assert_eq!(layer.content(), LayerContent::Count); + assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar); + assert_eq!(layer.evidence_kind(), crate::mphf_layer::EvidenceKind::Exact); +} + +#[test] +fn presence_layer_reports_presence_content_and_columnar_storage() { + set_k(4); + let dir = tempdir().unwrap(); + write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]); + Layer::::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| { + (kmer.raw().wrapping_add(g as u64)) % 2 == 0 + }).unwrap(); + let layer = Layer::::open(dir.path(), &IndexMode::Exact).unwrap(); + + assert_eq!(layer.content(), LayerContent::Presence); + assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar); +} + #[test] fn enumerate_kmers_is_stable_across_calls() { set_k(4); diff --git a/src/obilayeredmap/src/tests/map.rs b/src/obilayeredmap/src/tests/map.rs index d8e2ce81..276597ed 100644 --- a/src/obilayeredmap/src/tests/map.rs +++ b/src/obilayeredmap/src/tests/map.rs @@ -1,7 +1,10 @@ use super::*; -use obicompactvec::PersistentCompactIntMatrix; +use obicompactvec::{pack_bit_matrix, pack_sparse_bit_matrix, PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind}; use obikseq::{set_k, Sequence as _, Unitig}; +use obiskio::DEFAULT_BLOCK_BITS; +use crate::layer::LayerContent; use crate::meta::IndexMode; +use crate::mphf_layer::EvidenceKind; use tempfile::tempdir; fn push_unitigs_and_layer( @@ -96,3 +99,121 @@ fn push_layer_from_map_convenience() { let (_, hit) = map.query(canonical(b"AAAA")).unwrap(); assert_eq!(hit.data[0], 10); } + +// ── detect_layer_content / detect_layer_storage / detect_layer_evidence ──── +// +// All built directly on disk (bypassing `LayeredMap::push_layer`, which +// only exists for modes 1 and 2 — mode 3/presence has no `push_layer`), +// then reopened as `LayeredMap<()>` — `()`'s `LayerData::open` never reads +// the matrix, so it's the right handle for exercising the disk-probing +// `detect_layer_*` methods regardless of what content/storage the layer +// underneath actually is. + +fn write_unitigs_at(dir: &Path, seqs: &[&[u8]]) { + fs::create_dir_all(dir).unwrap(); + let mut w = UnitigFileWriter::create(&dir.join(crate::layer::UNITIGS_FILE)).unwrap(); + for s in seqs { + w.write(&Unitig::from_ascii(s)).unwrap(); + } + w.close().unwrap(); +} + +/// Build a one-layer partition root with a presence layer at `layer_0`, +/// save `meta.json`, and reopen it as `LayeredMap<()>`. +fn presence_partition(seqs: &[&[u8]], n_genomes: usize) -> (tempfile::TempDir, LayeredMap<()>) { + let dir = tempdir().unwrap(); + let mode = IndexMode::Exact; + let layer0 = layer_dir(dir.path(), 0); + write_unitigs_at(&layer0, seqs); + Layer::::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| { + (kmer.raw().wrapping_add(g as u64)) % 2 == 0 + }).unwrap(); + PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap(); + let map = LayeredMap::<()>::open(dir.path()).unwrap(); + (dir, map) +} + +#[test] +fn detect_layer_content_count() { + set_k(4); + let dir = tempdir().unwrap(); + let mut map = LayeredMap::::create(dir.path(), IndexMode::Exact).unwrap(); + push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3); + assert_eq!(map.detect_layer_content(0), LayerContent::Count); +} + +#[test] +fn detect_layer_content_presence() { + set_k(4); + let (_dir, map) = presence_partition(&[b"AAATCTA", b"CTTCGCC", b"TGATACG"], 3); + assert_eq!(map.detect_layer_content(0), LayerContent::Presence); +} + +#[test] +fn detect_layer_storage_columnar_then_packed_then_sparse() { + set_k(4); + let (dir, map) = presence_partition(&[b"AAATCTA", b"CTTCGCC", b"TGATACG"], 3); + let presence_dir = layer_dir(dir.path(), 0).join("presence"); + + assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Columnar); + + pack_bit_matrix(&presence_dir).unwrap(); + assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Packed); + + pack_sparse_bit_matrix(&presence_dir).unwrap(); + assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Sparse); +} + +#[test] +fn detect_layer_storage_implicit() { + set_k(4); + let dir = tempdir().unwrap(); + let mode = IndexMode::Exact; + let layer0 = layer_dir(dir.path(), 0); + write_unitigs_at(&layer0, &[b"AAAACGT"]); + // Mode-1 build: MPHF + evidence + `layer_meta.json`, no matrix at all — + // must read back as `Presence`/`Implicit`, never a third "empty" state. + Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap(); + PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap(); + let map = LayeredMap::<()>::open(dir.path()).unwrap(); + + assert_eq!(map.detect_layer_content(0), LayerContent::Presence); + assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Implicit); +} + +#[test] +fn detect_layer_storage_count_columnar_then_packed() { + set_k(4); + let dir = tempdir().unwrap(); + let mut map = LayeredMap::::create(dir.path(), IndexMode::Exact).unwrap(); + push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3); + drop(map); + + let map = LayeredMap::<()>::open(dir.path()).unwrap(); + assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Columnar); + + obicompactvec::pack_compact_int_matrix(&layer_dir(dir.path(), 0).join("counts")).unwrap(); + assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Packed); +} + +#[test] +fn detect_layer_evidence_exact_vs_approx() { + set_k(4); + + let dir = tempdir().unwrap(); + let layer0 = layer_dir(dir.path(), 0); + write_unitigs_at(&layer0, &[b"AAAACGT"]); + Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap(); + PartitionMeta { n_layers: 1, mode: IndexMode::Exact }.save(dir.path()).unwrap(); + let map = LayeredMap::<()>::open(dir.path()).unwrap(); + assert_eq!(map.detect_layer_evidence(0).unwrap(), EvidenceKind::Exact); + + let approx = IndexMode::Approx { b: 8, z: 1 }; + let dir2 = tempdir().unwrap(); + let layer0b = layer_dir(dir2.path(), 0); + write_unitigs_at(&layer0b, &[b"AAAACGT"]); + Layer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap(); + PartitionMeta { n_layers: 1, mode: approx }.save(dir2.path()).unwrap(); + let map2 = LayeredMap::<()>::open(dir2.path()).unwrap(); + assert_eq!(map2.detect_layer_evidence(0).unwrap(), EvidenceKind::Approx); +}