diff --git a/DevDocMD/implementation/partition_layer_cache.md b/DevDocMD/implementation/partition_layer_cache.md index 7b3e087c..cb5e1558 100644 --- a/DevDocMD/implementation/partition_layer_cache.md +++ b/DevDocMD/implementation/partition_layer_cache.md @@ -133,14 +133,35 @@ needed — `obikindex → obikpartitionner` stays the same direction as before. Full workspace test suite green (0 failed) after, including all 27 `obikphylo::siblings` tests. -Still open: (1)/(2) themselves — `AnyLayer` (or whatever it ends up named; -`AnyLayer` was rejected as a placeholder, no replacement chosen yet) and -`KmerPartition` (singular, one partition's open layers) are not built yet. -`obikphylo::siblings::cache::{Mat, PartitionCache}` and -`obikpartitionner::query_layer` (now `obikindex::query_layer`)'s -`QueryLayer` still each independently bundle MPHF+matrix — unchanged by -this restructuring, which was purely about *where* code lives, not about -building the heterogeneous-layer cache itself. +## (1) done (2026-08-20): `Layer` is now the heterogeneous handle, `Mat` is gone + +Resolved the naming question left open above. `Layer` (the old +generic/monomorphic type) renamed to `TypedLayer` throughout +(`obilayeredmap`, `obikindex`, `obikphylo` — 12 files, mechanical) to free +`Layer` for the type that's actually meant to be everyone's default +handle. `obilayeredmap::content_layer::Layer` (re-exported at the crate +root) is that type — `Count(TypedLayer)`/ +`Presence(TypedLayer)`, `Layer::open` doing the same +disk probe `Mat::open` used to, `find_slot`/`index_batch`/`n_cols`/ +`fill_sub_matrix_carries` dispatching per variant exactly as `Mat` did. + +`obikphylo::siblings::cache::Mat` deleted outright — `PartitionCache` now +holds `Vec>` directly. The one sibling-specific +method `Mat` carried (`iter_minorants_batch`) is not on `obilayeredmap:: +Layer` (phylo concepts don't belong in `obilayeredmap`) — it's an +`impl SiblingLayerExt for obilayeredmap::Layer` in `iter.rs`, dispatching +to each variant's existing `impl SiblingLayerExt for +TypedLayer`. + +Full workspace suite green (0 failed) after, including all 27 +`obikphylo::siblings` tests. + +Still not built: (2) — `KmerPartition` (singular, one partition's open +`Vec`) and a multi-partition cache in `obikpartitionner` to replace +`obikphylo::siblings::cache::PartitionCache` and `obikindex::query_layer`'s +still-separate `QueryLayer` (which still independently bundles MPHF+matrix, +2-way not using `Layer` at all). Both remaining consumers now sit one +`Layer::open` call away from unifying onto (2) once it exists. ## The problem diff --git a/src/obikindex/src/graph_pipeline.rs b/src/obikindex/src/graph_pipeline.rs index c1c8302c..5b49a3c1 100644 --- a/src/obikindex/src/graph_pipeline.rs +++ b/src/obikindex/src/graph_pipeline.rs @@ -10,7 +10,7 @@ use obipipeline::{ use obidebruinj::GraphDeBruijn; use obikseq::CanonicalKmer; -use obilayeredmap::{IndexMode, Layer}; +use obilayeredmap::{IndexMode, TypedLayer}; use obiskio::{SKError, SKResult}; use crate::common::olm_to_sk; @@ -125,7 +125,7 @@ pub(crate) fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> SKRe let n_kmers = g.len(); g.compute_degrees_and_mark_starts(); std::fs::create_dir_all(layer_dir)?; - let mut uw = Layer::<()>::unitig_writer(layer_dir).map_err(|e| olm_to_sk(e, "graph pipeline"))?; + let mut uw = TypedLayer::<()>::unitig_writer(layer_dir).map_err(|e| olm_to_sk(e, "graph pipeline"))?; g.try_for_each_unitig(|unitig| uw.write(unitig))?; uw.close()?; drop(g); @@ -134,7 +134,7 @@ pub(crate) fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> SKRe // ── materialize_layer ───────────────────────────────────────────────────────── -/// Phase 2 (full): write_graph_as_unitigs + `Layer::<()>::build`. +/// Phase 2 (full): write_graph_as_unitigs + `TypedLayer::<()>::build`. /// /// Returns n_kmers. pub(crate) fn materialize_layer( @@ -145,7 +145,7 @@ pub(crate) fn materialize_layer( ) -> SKResult { let n = write_graph_as_unitigs(g, layer_dir)?; debug!("materialize_layer: unitigs written ({n} kmers), building MPHF"); - Layer::<()>::build(layer_dir, block_bits, evidence) + TypedLayer::<()>::build(layer_dir, block_bits, evidence) .map_err(|e| olm_to_sk(e, "graph pipeline"))?; debug!("materialize_layer: MPHF build done"); Ok(n) diff --git a/src/obikindex/src/index_layer.rs b/src/obikindex/src/index_layer.rs index 46b8d842..a01aa13d 100644 --- a/src/obikindex/src/index_layer.rs +++ b/src/obikindex/src/index_layer.rs @@ -6,7 +6,7 @@ use epserde::prelude::*; use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec}; use obidebruinj::GraphDeBruijn; use obilayeredmap::meta::PartitionMeta; -use obilayeredmap::{IndexMode, layer::Layer}; +use obilayeredmap::{IndexMode, layer::TypedLayer}; use obiskio::{SKError, SKFileMeta, SKFileReader}; use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64}; @@ -96,7 +96,7 @@ impl KmerIndex { let n_kmers = if with_counts { let n = write_graph_as_unitigs(g, &layer_dir)?; - Layer::::build(&layer_dir, block_bits, mode, |kmer| { + TypedLayer::::build(&layer_dir, block_bits, mode, |kmer| { match (&mphf1_opt, &counts1_opt) { (Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())), _ => 1, diff --git a/src/obikindex/src/merge_layer/mod.rs b/src/obikindex/src/merge_layer/mod.rs index 9f96f73d..4ef0a731 100644 --- a/src/obikindex/src/merge_layer/mod.rs +++ b/src/obikindex/src/merge_layer/mod.rs @@ -20,7 +20,7 @@ use tracing::debug; use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder}; use obikseq::CanonicalKmer; -use obilayeredmap::{IndexMode, Layer, LayeredMap, MphfOnly, layer_dir}; +use obilayeredmap::{IndexMode, TypedLayer, LayeredMap, MphfOnly, layer_dir}; use obiskio::{SKError, SKResult, UnitigFileReader}; use crate::common::{ColBuilder, load_meta, olm_to_sk}; @@ -232,7 +232,7 @@ impl KmerIndex { // (all slots true — every kmer in those layers belongs to genome_0). if n_dst_genomes == 1 && mode == MergeMode::Presence { for l in 0..n_dst_layers { - Layer::<()>::init_presence_matrix( + TypedLayer::<()>::init_presence_matrix( &layer_dir(&dst_index_dir, l), dst_map.layer(l).n(), ) diff --git a/src/obikindex/src/reindex.rs b/src/obikindex/src/reindex.rs index 6265e665..2d3dff3b 100644 --- a/src/obikindex/src/reindex.rs +++ b/src/obikindex/src/reindex.rs @@ -1,4 +1,4 @@ -use obilayeredmap::{IndexMode, layer::Layer}; +use obilayeredmap::{IndexMode, layer::TypedLayer}; use obisys::{Reporter, Stage, progress_bar}; use std::fs; use std::path::Path; @@ -90,10 +90,10 @@ fn reindex_partition( fn reindex_layer(layer_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> { match target { IndexMode::Exact => { - Layer::<()>::build_exact_evidence(layer_dir, block_bits).map_err(olm_to_oki)?; + TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits).map_err(olm_to_oki)?; } IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => { - Layer::<()>::build_approx_evidence(layer_dir, *b, *z).map_err(olm_to_oki)?; + TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z).map_err(olm_to_oki)?; } } remove_stale_evidence(layer_dir, target) diff --git a/src/obikphylo/src/siblings/cache.rs b/src/obikphylo/src/siblings/cache.rs index 06c29676..1381fbd0 100644 --- a/src/obikphylo/src/siblings/cache.rs +++ b/src/obikphylo/src/siblings/cache.rs @@ -1,18 +1,11 @@ use rayon::prelude::*; -use std::path::Path; - -use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obikseq::CanonicalKmer; -use obilayeredmap::meta::IndexMode; -use obilayeredmap::{Layer, OLMResult}; +use obilayeredmap::Layer; use obisys::progress_bar; use obikindex::{KmerIndex, OKIResult}; -use super::SiblingAnnex; -use super::iter::SiblingLayerExt; - /// Every partition's already-open layers, built **once** for the whole /// `build_sibling_annex` run and shared (read-only) across every lookup, in /// every source layer, for the rest of the run — not reopened/re-mmap'd per @@ -31,119 +24,14 @@ use super::iter::SiblingLayerExt; /// partition for the entire run, regardless of how many source layers or /// lookups follow. /// -/// One `Layer` per layer (MPHF + matrix bundled), not a separate -/// `MphfLayer` and a separate `PersistentCompactIntMatrix`/ -/// `PersistentBitMatrix` in parallel arrays — `obilayeredmap::Layer` already -/// *is* that bundle, with `find_slot` (MPHF-only, no data read), -/// `n_cols`/`sub_matrix`/`fill_sub_matrix` (batched, sorted-internally -/// column access) on top of it. Reinventing that pairing here would just be -/// going back through the low-level pieces `Layer` already assembles. -pub(super) enum Mat { - Count(Layer), - /// Dense *or* sparse (`pack --sparse`d) presence — `PersistentBitMatrix` - /// itself is a 4-way enum (`Columnar`/`Packed`/`Sparse`/`Implicit`) that - /// already auto-detects sparse storage in its own `open()` (checks - /// `presence/sparse_meta.json`) and dispatches every method - /// (`row`/`fill_row`/`nonzero_iter`/...) across all four internally — - /// see `DevDocMD/implementation/partition_layer_cache.md`. A separate - /// `SparsePresence(Layer)` arm used to exist - /// here, opened via its own `presence/is_multi.prsb` check; it - /// predated `PersistentBitMatrix` growing native sparse support and - /// was pure duplication by the time it was removed — every method - /// below dispatched it identically to this arm. - Presence(Layer), -} - -impl Mat { - /// Open one layer's matrix, auto-detecting count vs. presence from what - /// is actually on disk — the single source of truth for *every* caller - /// that opens a layer's own matrix (this module's cross-partition - /// [`PartitionCache::build`] and `family_scan::scan_layer_families`'s - /// own-layer lookup alike), so the two can never disagree. Sparse vs. - /// dense presence storage is `PersistentBitMatrix::open`'s own concern - /// (see the [`Presence`](Mat::Presence) variant's docs), not decided - /// here. - pub(super) fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult { - if with_counts && layer_dir.join("counts").exists() { - return Layer::::open(layer_dir, mode).map(Mat::Count); - } - Layer::::open(layer_dir, mode).map(Mat::Presence) - } - - fn find_slot(&self, kmer: CanonicalKmer) -> Option { - match self { - Mat::Count(l) => l.find_slot(kmer), - Mat::Presence(l) => l.find_slot(kmer), - } - } - - /// Raw MPHF batch lookup: kmer → slot, no membership check — for - /// callers that already know every kmer is a member of *this* layer - /// (e.g. it came from this layer's own `iter_minorants_batch`), so the - /// evidence check `find_slot`/`find` would perform is redundant work. - /// See `DevDocMD/architecture/siblings.md`: iteration-pipeline kmers use - /// `index`, never `find`. - pub(super) fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec { - match self { - Mat::Count(l) => l.index_batch(kmers), - Mat::Presence(l) => l.index_batch(kmers), - } - } - - /// This layer's own `SiblingLayerExt::iter_minorants_batch` — dispatch - /// only, both arms return the same concrete `MinorantBatchIter` (it - /// doesn't depend on `D`), so no boxing is needed. - pub(super) fn iter_minorants_batch( - &self, - annex: std::sync::Arc, - batch_size: usize, - ) -> super::iter::MinorantBatchIter { - match self { - Mat::Count(l) => l.iter_minorants_batch(annex, batch_size), - Mat::Presence(l) => l.iter_minorants_batch(annex, batch_size), - } - } - - pub(super) fn n_cols(&self) -> usize { - match self { - Mat::Count(l) => l.n_cols(), - Mat::Presence(l) => l.n_cols(), - } - } - - /// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` = - /// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must - /// have one entry per genome column, each resized to `slots.len()`. - /// - /// Delegates entirely to `Layer::fill_sub_matrix`, which sorts - /// `slots` once internally for a sequential mmap sweep per column, then - /// restores the original order — the same discipline this call site - /// (and `find_presence_batch`) used to hand-roll with its own sort + - /// genome-major loop. `Count` still needs one intermediate - /// `Vec>` fetch (the underlying store only has an int - /// sub-matrix, not a bool one), converted to presence (`!= 0`) in place - /// — the sort/sequential-access win is unaffected, just one extra - /// allocation pass over already-in-hand data. - pub(super) fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec]) { - match self { - Mat::Presence(l) => l.fill_sub_matrix(slots, out), - Mat::Count(l) => { - let mut counts: Vec> = out.iter().map(|_| Vec::new()).collect(); - l.fill_sub_matrix(slots, &mut counts); - for (o, c) in out.iter_mut().zip(counts.iter()) { - o.clear(); - o.extend(c.iter().map(|&v| v != 0)); - } - } - } - } -} - +/// `mats[partition][layer]` holds `obilayeredmap::Layer` — the +/// format-erased `Count`/`Presence` handle, not a sibling-specific type: +/// this module used to bundle its own `Mat` enum here, duplicating exactly +/// what `Layer` now does one crate down (see +/// `DevDocMD/implementation/partition_layer_cache.md`). Its +/// sibling-specific extension (`iter_minorants_batch`) lives in `iter.rs`. pub(super) struct PartitionCache { - /// `mats[partition][layer]` = that partition's opened layers. Used by - /// both [`obikindex::KmerIndex::build_sibling_annex`] and - /// [`obikindex::KmerIndex::sibling_annex_stats`]. - mats: Vec>, + mats: Vec>, /// Whether every `FamilyMask` field in this index's sibling annexes can /// be trusted as a real layer index (`n_layers <= 7`, the same decision /// `build_layer_sibling_annex` makes when writing them) — computed once @@ -163,9 +51,9 @@ impl PartitionCache { with_counts: bool, ) -> OKIResult { let pb = progress_bar("open_partitions", n_parts as u64, "partitions"); - let built: Vec<(Vec, usize)> = (0..n_parts) + let built: Vec<(Vec, usize)> = (0..n_parts) .into_par_iter() - .map(|part| -> OKIResult<(Vec, usize)> { + .map(|part| -> OKIResult<(Vec, usize)> { let index_dir = index.index_dir(part); if !index_dir.exists() { pb.inc(1); @@ -174,7 +62,7 @@ impl PartitionCache { let meta = index.partition_meta(part)?; let mut mats = Vec::with_capacity(meta.n_layers); for l in 0..meta.n_layers { - let Ok(mat) = Mat::open(&index.layer_dir(part, l), &meta.mode, with_counts) + let Ok(mat) = Layer::open(&index.layer_dir(part, l), &meta.mode, with_counts) else { continue; }; @@ -331,7 +219,7 @@ impl PartitionCache { /// `hits` gets its slots (evidence-probed vs. trusted `index_batch`) — this /// is everything after that. fn resolve_layer_hits( - mat: &Mat, + mat: &Layer, hits: &[(usize, usize, u8)], n_genomes: usize, on_hit: &mut impl FnMut(usize, u8, usize), diff --git a/src/obikphylo/src/siblings/family_scan.rs b/src/obikphylo/src/siblings/family_scan.rs index 5a45b6c9..ee55e643 100644 --- a/src/obikphylo/src/siblings/family_scan.rs +++ b/src/obikphylo/src/siblings/family_scan.rs @@ -60,9 +60,11 @@ use obipipeline::{ThrottleGuard, throttle}; use obikindex::{OKIError, OKIResult}; use obikindex::KmerIndex; -use super::cache::{Mat, PartitionCache}; +use obilayeredmap::Layer; + +use super::cache::PartitionCache; use super::helpers::central_base; -use super::iter::SiblingEntry; +use super::iter::{SiblingEntry, SiblingLayerExt}; use super::{olm_to_ok, FamilyMask, SiblingAnnex, ANNEX_FILE_NAME}; /// Families per batch — see the module docs for the memory-vs-per-partition- @@ -129,10 +131,10 @@ pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult> { /// generating this layer's batches — opened once, not per batch. No `cache` /// here: generation never touches the cross-partition cache, only this /// layer's own already-open matrix. No separate MPHF/`slot_kmer` either — -/// `mat` (a `Layer`) already bundles the MPHF, and each `SiblingEntry` +/// `mat` (a `TypedLayer`) already bundles the MPHF, and each `SiblingEntry` /// arrives with its kmer and mask already in hand from `iter_minorants_batch`. struct LayerCtx { - mat: Mat, + mat: Layer, n_parts: usize, n_genomes: usize, n_cols: usize, @@ -193,7 +195,7 @@ pub(super) fn scan_layer_families( let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?; let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?); - let mat = Mat::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?; + let mat = Layer::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?; let n_cols = mat.n_cols().min(n_genomes); let ctx = Arc::new(LayerCtx { mat, n_parts, n_genomes, n_cols, k }); diff --git a/src/obikphylo/src/siblings/iter.rs b/src/obikphylo/src/siblings/iter.rs index b2e948ca..a3a7cc2c 100644 --- a/src/obikphylo/src/siblings/iter.rs +++ b/src/obikphylo/src/siblings/iter.rs @@ -1,5 +1,5 @@ //! Phylo/sibling-domain iteration over a layer — an extension trait, not a -//! new field on `MphfLayer`/`Layer`: "family"/"minorant" are phylo +//! new field on `MphfLayer`/`TypedLayer`: "family"/"minorant" are phylo //! concepts, `obilayeredmap` stays kmer/slot-mapping only (see //! `DevDocMD/architecture/siblings.md`). //! @@ -29,7 +29,7 @@ use std::sync::Arc; use obikseq::CanonicalKmer; -use obilayeredmap::{KmerIter, Layer, LayerData}; +use obilayeredmap::{KmerIter, TypedLayer, LayerData}; use super::{FamilyMask, SiblingAnnex}; @@ -128,10 +128,10 @@ fn collect_batch(inner: &mut I, batch_size: usize) -> Option` — the extension that +/// Adds phylo/sibling iteration to any `TypedLayer` — the extension that /// turns a plain layer into a "sibling layer". Generic over `D` /// (`LayerData`) rather than implemented once per matrix kind: kmer -/// iteration doesn't depend on the data payload, and `Layer` already +/// iteration doesn't depend on the data payload, and `TypedLayer` already /// delegates `iter_kmers`/`index`/`index_batch` to its inner MPHF for every /// `D` — reusing that instead of going back through a separate `MphfLayer`. pub trait SiblingLayerExt { @@ -154,7 +154,7 @@ pub trait SiblingLayerExt { fn iter_minorants_batch(&self, annex: Arc, batch_size: usize) -> MinorantBatchIter; } -impl SiblingLayerExt for Layer { +impl SiblingLayerExt for TypedLayer { fn iter_siblings(&self, annex: Arc) -> SiblingIter { SiblingIter { kmers: self.iter_kmers(), annex, order: 0 } } @@ -171,3 +171,39 @@ impl SiblingLayerExt for Layer { MinorantBatchIter { inner: self.iter_minorants(annex), batch_size } } } + +/// Same extension, over `obilayeredmap::Layer` (the format-erased +/// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch +/// only, both arms return the same concrete iterator types (they don't +/// depend on which `D` is inside), so no boxing is needed. `obilayeredmap` +/// itself can't implement this: "family"/"minorant" are phylo concepts, it +/// stays kmer/slot-mapping only (see the module docs above). +impl SiblingLayerExt for obilayeredmap::Layer { + fn iter_siblings(&self, annex: Arc) -> SiblingIter { + match self { + obilayeredmap::Layer::Count(l) => l.iter_siblings(annex), + obilayeredmap::Layer::Presence(l) => l.iter_siblings(annex), + } + } + + fn iter_siblings_batch(&self, annex: Arc, batch_size: usize) -> SiblingBatchIter { + match self { + obilayeredmap::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size), + obilayeredmap::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size), + } + } + + fn iter_minorants(&self, annex: Arc) -> MinorantIter { + match self { + obilayeredmap::Layer::Count(l) => l.iter_minorants(annex), + obilayeredmap::Layer::Presence(l) => l.iter_minorants(annex), + } + } + + fn iter_minorants_batch(&self, annex: Arc, batch_size: usize) -> MinorantBatchIter { + match self { + obilayeredmap::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size), + obilayeredmap::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size), + } + } +} diff --git a/src/obikphylo/src/siblings/tests.rs b/src/obikphylo/src/siblings/tests.rs index 2faba0cc..46e990b9 100644 --- a/src/obikphylo/src/siblings/tests.rs +++ b/src/obikphylo/src/siblings/tests.rs @@ -156,9 +156,9 @@ fn sibling_annex_works_after_pack_sparse() { // `pack --sparse` (`KmerIndex::pack_matrices(true)`) run on the merged // index before building the sibling annex — proves `PartitionCache`'s // sparse-detection (`Mat::SparsePresence`, gated on `presence/ - // is_multi.prsb`) and the generic `Layer` methods it relies on + // is_multi.prsb`) and the generic `TypedLayer` methods it relies on // actually round-trip through the real build pipeline, not just the - // unit-level `Layer` tests in + // unit-level `TypedLayer` tests in // `obilayeredmap`. let dir = tempdir().unwrap(); let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); diff --git a/src/obilayeredmap/src/content_layer.rs b/src/obilayeredmap/src/content_layer.rs new file mode 100644 index 00000000..aa8144a1 --- /dev/null +++ b/src/obilayeredmap/src/content_layer.rs @@ -0,0 +1,113 @@ +//! [`Layer`] — the format-erased layer handle every caller outside this +//! crate should reach for. `TypedLayer` (`layer.rs`) is monomorphic: a +//! `Vec>` needs one `D` fixed at compile time for every +//! element, which real partitions don't respect (`with_counts` can differ +//! layer to layer across merges; see +//! `DevDocMD/implementation/partition_layer_cache.md`). `Layer` is the +//! sum type that lets a partition's layers be held together regardless — +//! `Count`/`Presence` each wrap the one concrete `TypedLayer` that +//! content implies, chosen by [`Layer::open`]'s own disk probe. +//! +//! This is exactly the shape `obikphylo::siblings::cache::Mat` used to +//! reimplement locally, minus its one sibling-specific method +//! (`iter_minorants_batch`, which stays an extension trait over there — +//! `obilayeredmap` has no business knowing about sibling annexes). + +use std::path::Path; + +use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; +use obikseq::CanonicalKmer; + +use crate::error::OLMResult; +use crate::layer::{LayerContent, TypedLayer, COUNTS_DIR}; +use crate::meta::IndexMode; +use crate::mphf_layer::EvidenceKind; + +/// One layer, its content (`Count`/`Presence`) resolved at open time rather +/// than at compile time — see the module docs. +pub enum Layer { + Count(TypedLayer), + Presence(TypedLayer), +} + +impl Layer { + /// Open one layer, auto-detecting count vs. presence from what is + /// actually on disk (`counts/` present and wanted, else presence) — the + /// single source of truth every caller that opens a layer's own matrix + /// should share, so two callers can never disagree about which format a + /// given layer is. Dense vs. sparse presence storage is + /// `PersistentBitMatrix::open`'s own concern (it's a 4-way + /// `Columnar`/`Packed`/`Sparse`/`Implicit` enum internally), not decided + /// here. + pub fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult { + if with_counts && layer_dir.join(COUNTS_DIR).exists() { + return TypedLayer::::open(layer_dir, mode).map(Layer::Count); + } + TypedLayer::::open(layer_dir, mode).map(Layer::Presence) + } + + pub fn content(&self) -> LayerContent { + match self { + Layer::Count(_) => LayerContent::Count, + Layer::Presence(_) => LayerContent::Presence, + } + } + + pub fn evidence_kind(&self) -> EvidenceKind { + match self { + Layer::Count(l) => l.evidence_kind(), + Layer::Presence(l) => l.evidence_kind(), + } + } + + pub fn n(&self) -> usize { + match self { + Layer::Count(l) => l.n(), + Layer::Presence(l) => l.n(), + } + } + + pub fn find_slot(&self, kmer: CanonicalKmer) -> Option { + match self { + Layer::Count(l) => l.find_slot(kmer), + Layer::Presence(l) => l.find_slot(kmer), + } + } + + /// Raw MPHF batch lookup: kmer → slot, no membership check — for + /// callers that already know every kmer is a member of *this* layer, so + /// the evidence check `find_slot`/`find` would perform is redundant. + pub fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec { + match self { + Layer::Count(l) => l.index_batch(kmers), + Layer::Presence(l) => l.index_batch(kmers), + } + } + + pub fn n_cols(&self) -> usize { + match self { + Layer::Count(l) => l.n_cols(), + Layer::Presence(l) => l.n_cols(), + } + } + + /// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` = + /// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must + /// have one entry per genome column, each resized to `slots.len()`. + /// `Count` needs one intermediate `Vec>` fetch (the underlying + /// store only has an int sub-matrix, not a bool one), converted to + /// presence (`!= 0`) in place. + pub fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec]) { + match self { + Layer::Presence(l) => l.fill_sub_matrix(slots, out), + Layer::Count(l) => { + let mut counts: Vec> = out.iter().map(|_| Vec::new()).collect(); + l.fill_sub_matrix(slots, &mut counts); + for (o, c) in out.iter_mut().zip(counts.iter()) { + o.clear(); + o.extend(c.iter().map(|&v| v != 0)); + } + } + } + } +} diff --git a/src/obilayeredmap/src/layer.rs b/src/obilayeredmap/src/layer.rs index eea8ed81..142b3a1f 100644 --- a/src/obilayeredmap/src/layer.rs +++ b/src/obilayeredmap/src/layer.rs @@ -27,7 +27,7 @@ pub trait LayerData: Sized { fn read(&self, slot: usize) -> Self::Item; } -/// Layer directory path for layer `i` within a partition's index root — the +/// TypedLayer directory path for layer `i` within a partition's index root — the /// single source of truth for the on-disk `layer_N` naming convention /// (mirrors what `LayeredMap::open`/`push_layer` already use internally). /// @@ -43,7 +43,7 @@ pub fn layer_dir(root: &Path, i: usize) -> PathBuf { /// Opens layer `i`'s data only, skipping the MPHF — for callers that only /// need matrix-level operations (distance traits, column weights, group /// filters, sub-matrix extraction) and never look up a kmer for this layer. -/// `Layer::open` always pays for the MPHF too (and doesn't expose `data` +/// `TypedLayer::open` always pays for the MPHF too (and doesn't expose `data` /// once open), so it's the wrong tool for these; this is the other half of /// the same `D::open(layer_dir)` call, without the MPHF alongside it. /// @@ -89,7 +89,7 @@ impl LayerData for PersistentSparseBitMatrix { /// What a layer's data matrix represents — orthogonal to *how* it's stored /// on disk (`obicompactvec::StorageKind`'s concern). Only meaningful for -/// `D` that actually carry a matrix: `Layer<()>` (mode 1, set membership, +/// `D` that actually carry a matrix: `TypedLayer<()>` (mode 1, set membership, /// no matrix at all) has neither a `LayerContent` nor a `content()` method /// — it's a write-time-only state, never a queryable content. Once a layer /// is closed, "no matrix file" reads back as `Presence` via @@ -115,7 +115,7 @@ impl LayerContent { } /// Implemented only by `D` that carry a real matrix — gates -/// [`Layer::content`](Layer::content) so `Layer<()>` doesn't get it. +/// [`TypedLayer::content`](TypedLayer::content) so `TypedLayer<()>` doesn't get it. pub trait HasLayerContent { const CONTENT: LayerContent; } @@ -135,7 +135,7 @@ impl HasLayerContent for PersistentSparseBitMatrix { // ── StorageKind passthrough ──────────────────────────────────────────────────── /// Implemented only by `D` that expose their own `storage_kind()` — gates -/// [`Layer::storage_kind`]. `PersistentSparseBitMatrix` has a single, +/// [`TypedLayer::storage_kind`]. `PersistentSparseBitMatrix` has a single, /// fixed on-disk format (no `Columnar`/`Packed`/`Implicit` variants of its /// own), so it's deliberately not given an impl: there's nothing to report /// beyond what `content()` already says. @@ -157,7 +157,7 @@ impl HasStorageKind for PersistentBitMatrix { // ── Structures ──────────────────────────────────────────────────────────────── -pub struct Layer { +pub struct TypedLayer { mphf: MphfLayer, data: D, } @@ -169,7 +169,7 @@ pub struct Hit { // ── Common read path ────────────────────────────────────────────────────────── -impl Layer { +impl TypedLayer { pub fn open(path: &Path, mode: &IndexMode) -> OLMResult { let mphf = MphfLayer::open(path, mode)?; let data = D::open(path)?; @@ -249,7 +249,7 @@ impl Layer { } } -impl Layer { +impl TypedLayer { /// This already-open layer's content (`Count`/`Presence`) — a /// compile-time fact about `D`, not a runtime read. pub const fn content(&self) -> LayerContent { @@ -257,7 +257,7 @@ impl Layer { } } -impl Layer { +impl TypedLayer { /// This already-open layer's storage format — delegates to `D`'s own /// `storage_kind()`, reading a discriminant already in memory. pub fn storage_kind(&self) -> obicompactvec::StorageKind { @@ -267,7 +267,7 @@ impl Layer { // ── Mode 1 — set membership ─────────────────────────────────────────────────── -impl Layer<()> { +impl TypedLayer<()> { pub fn build(out_dir: &Path, block_bits: u8, mode: &IndexMode) -> OLMResult { MphfLayer::build(out_dir, block_bits, mode, &mut |_, _| Ok(())) } @@ -284,7 +284,7 @@ impl Layer<()> { // ── Mode 2 — count matrix ───────────────────────────────────────────────────── -impl Layer { +impl TypedLayer { pub fn build( out_dir: &Path, block_bits: u8, @@ -317,7 +317,7 @@ impl Layer { // ── Mode 2 — count matrix column append ────────────────────────────────────── -impl Layer { +impl TypedLayer { pub fn append_genome_column( layer_dir: &Path, value_of: impl Fn(usize) -> u32, @@ -357,14 +357,14 @@ impl Layer { // // `n_cols`/`sub_matrix`/`fill_sub_matrix` are identical in shape for any // `D` satisfying `BinaryMatrix` — genuinely generic (not just two -// near-identical impl blocks) so `Layer` gets -// them for free, matching `Layer` at the same call +// near-identical impl blocks) so `TypedLayer` gets +// them for free, matching `TypedLayer` at the same call // sites (see `obikphylo::siblings::cache::Mat`). Construction // (`append_genome_column`/`build_presence`) stays `PersistentBitMatrix`-only // below: sparse matrices aren't built column-by-column, they're built // row-by-row from an already-built dense layer // (`PersistentSparseBitMatrixBuilder::build_from_dense`). -impl> + BinaryMatrix> Layer { +impl> + BinaryMatrix> TypedLayer { /// Number of genome columns in this layer's presence matrix — see /// `PersistentBitMatrix::n_cols`'s docs for the `Implicit` mono-genome /// special case (always reports `1`, regardless of the index's real @@ -395,7 +395,7 @@ impl> + BinaryMatrix> Layer { } } -impl Layer { +impl TypedLayer { pub fn append_genome_column( layer_dir: &Path, value_of: impl Fn(usize) -> bool, diff --git a/src/obilayeredmap/src/lib.rs b/src/obilayeredmap/src/lib.rs index 66387b27..3df4fa9d 100644 --- a/src/obilayeredmap/src/lib.rs +++ b/src/obilayeredmap/src/lib.rs @@ -1,3 +1,4 @@ +pub mod content_layer; pub mod error; pub mod evidence; pub mod fingerprint; @@ -7,8 +8,9 @@ pub mod map; pub mod meta; pub(crate) mod mphf_layer; +pub use content_layer::Layer; pub use error::{OLMError, OLMResult}; -pub use layer::{layer_dir, open_data, HasLayerContent, HasStorageKind, Hit, Layer, LayerContent, LayerData}; +pub use layer::{layer_dir, open_data, HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer}; pub use layered_store::LayeredStore; pub use map::LayeredMap; pub use meta::{IndexMode, PartitionMeta}; diff --git a/src/obilayeredmap/src/map.rs b/src/obilayeredmap/src/map.rs index 96d27975..891be1be 100644 --- a/src/obilayeredmap/src/map.rs +++ b/src/obilayeredmap/src/map.rs @@ -7,7 +7,7 @@ use obikseq::CanonicalKmer; use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS}; use crate::error::{OLMError, OLMResult}; -use crate::layer::{layer_dir, Hit, Layer, LayerContent, LayerData}; +use crate::layer::{layer_dir, Hit, TypedLayer, LayerContent, LayerData}; use crate::meta::{IndexMode, PartitionMeta}; use crate::mphf_layer::EvidenceKind; @@ -19,7 +19,7 @@ use crate::mphf_layer::EvidenceKind; pub struct LayeredMap { root: PathBuf, meta: PartitionMeta, - layers: Vec>, + layers: Vec>, } // ── Common methods ──────────────────────────────────────────────────────────── @@ -30,7 +30,7 @@ impl LayeredMap { pub fn open(root: &Path) -> OLMResult { let meta = PartitionMeta::load(root)?; let layers = (0..meta.n_layers) - .map(|i| Layer::::open(&layer_dir(root, i), &meta.mode)) + .map(|i| TypedLayer::::open(&layer_dir(root, i), &meta.mode)) .collect::>>()?; Ok(Self { root: root.to_owned(), meta, layers }) } @@ -44,18 +44,18 @@ impl LayeredMap { } pub fn n_layers(&self) -> usize { self.layers.len() } - pub fn layer(&self, i: usize) -> &Layer { &self.layers[i] } + pub fn layer(&self, i: usize) -> &TypedLayer { &self.layers[i] } pub fn mode(&self) -> &IndexMode { &self.meta.mode } - /// Layer `i`'s content (`Count`/`Presence`) — a lightweight disk probe, + /// TypedLayer `i`'s content (`Count`/`Presence`) — a lightweight disk probe, /// no MPHF/matrix opened. For picking a `D` before committing to - /// [`Layer::open`], or for reporting/diagnostics over an already-open + /// [`TypedLayer::open`], or for reporting/diagnostics over an already-open /// `LayeredMap` without re-opening every layer under a different `D`. pub fn detect_layer_content(&self, i: usize) -> LayerContent { LayerContent::detect(&layer_dir(&self.root, i)) } - /// Layer `i`'s storage format — a lightweight disk probe, no + /// TypedLayer `i`'s storage format — a lightweight disk probe, no /// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content). pub fn detect_layer_storage(&self, i: usize) -> OLMResult { let dir = layer_dir(&self.root, i); @@ -65,7 +65,7 @@ impl LayeredMap { } } - /// Layer `i`'s evidence mode — a lightweight disk probe, no + /// TypedLayer `i`'s evidence mode — a lightweight disk probe, no /// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content). pub fn detect_layer_evidence(&self, i: usize) -> OLMResult { EvidenceKind::detect(&layer_dir(&self.root, i)) @@ -81,13 +81,13 @@ impl LayeredMap { pub fn next_layer_writer(&self) -> OLMResult { let dir = layer_dir(&self.root, self.layers.len()); - Layer::::unitig_writer(&dir) + TypedLayer::::unitig_writer(&dir) } fn append_layer(&mut self) -> OLMResult<()> { let i = self.layers.len(); let dir = layer_dir(&self.root, i); - self.layers.push(Layer::::open(&dir, &self.meta.mode)?); + self.layers.push(TypedLayer::::open(&dir, &self.meta.mode)?); self.meta.n_layers = self.layers.len(); self.meta.save(&self.root)?; Ok(()) @@ -100,7 +100,7 @@ impl LayeredMap<()> { pub fn push_layer(&mut self) -> OLMResult { let i = self.layers.len(); let dir = layer_dir(&self.root, i); - Layer::<()>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode)?; + TypedLayer::<()>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode)?; self.append_layer()?; Ok(i) } @@ -112,7 +112,7 @@ impl LayeredMap { pub fn push_layer(&mut self, count_of: impl Fn(CanonicalKmer) -> u32) -> OLMResult { let i = self.layers.len(); let dir = layer_dir(&self.root, i); - Layer::::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode, count_of)?; + TypedLayer::::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode, count_of)?; self.append_layer()?; Ok(i) } diff --git a/src/obilayeredmap/src/tests/layer.rs b/src/obilayeredmap/src/tests/layer.rs index 31a3bdb7..e409e7b3 100644 --- a/src/obilayeredmap/src/tests/layer.rs +++ b/src/obilayeredmap/src/tests/layer.rs @@ -43,7 +43,7 @@ fn canonical_kmer_iter_matches_reader() { assert_eq!(from_iter, from_reader, "CanonicalKmerIter and UnitigFileReader disagree"); } -// ── Generic `Layer` over dense vs. sparse presence storage ─────────────── +// ── Generic `TypedLayer` over dense vs. sparse presence storage ─────────────── #[test] fn presence_layer_generic_over_sparse_matches_dense() { @@ -64,11 +64,11 @@ fn presence_layer_generic_over_sparse_matches_dense() { // kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be // biologically meaningful, just the same on both sides of the // dense/sparse comparison below. - Layer::::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| { + TypedLayer::::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| { (kmer.raw().wrapping_add(g as u64)) % 2 == 0 }).unwrap(); - let dense_layer = Layer::::open(dir.path(), &mode).unwrap(); + let dense_layer = TypedLayer::::open(dir.path(), &mode).unwrap(); assert!(dense_layer.n_cols() >= 1); // Build the sparse form directly into the same `presence/` dir the @@ -81,7 +81,7 @@ fn presence_layer_generic_over_sparse_matches_dense() { .close() .unwrap(); - let sparse_layer = Layer::::open(dir.path(), &mode).unwrap(); + let sparse_layer = TypedLayer::::open(dir.path(), &mode).unwrap(); // Same generic methods, same results, different concrete `D`. assert_eq!(dense_layer.n_cols(), sparse_layer.n_cols()); @@ -91,7 +91,7 @@ fn presence_layer_generic_over_sparse_matches_dense() { assert_eq!(dense_layer.sub_matrix(&slots), sparse_layer.sub_matrix(&slots)); // The matrix-agnostic, MPHF-only surface (from the generic - // `impl Layer`) must also agree: same kmer set, + // `impl TypedLayer`) must also agree: same kmer set, // looked up through either concrete `D`. for kmer in all_canonical_kmers(dir.path()) { assert_eq!(dense_layer.find_slot(kmer), sparse_layer.find_slot(kmer)); @@ -105,9 +105,9 @@ fn count_layer_reports_count_content_and_columnar_storage() { set_k(4); let dir = tempdir().unwrap(); write_unitigs(dir.path(), &[b"AAAACGT"]); - Layer::::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1) + TypedLayer::::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1) .unwrap(); - let layer = Layer::::open(dir.path(), &IndexMode::Exact).unwrap(); + let layer = TypedLayer::::open(dir.path(), &IndexMode::Exact).unwrap(); assert_eq!(layer.content(), LayerContent::Count); assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar); @@ -119,10 +119,10 @@ fn presence_layer_reports_presence_content_and_columnar_storage() { set_k(4); let dir = tempdir().unwrap(); write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]); - Layer::::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| { + TypedLayer::::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| { (kmer.raw().wrapping_add(g as u64)) % 2 == 0 }).unwrap(); - let layer = Layer::::open(dir.path(), &IndexMode::Exact).unwrap(); + let layer = TypedLayer::::open(dir.path(), &IndexMode::Exact).unwrap(); assert_eq!(layer.content(), LayerContent::Presence); assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar); diff --git a/src/obilayeredmap/src/tests/map.rs b/src/obilayeredmap/src/tests/map.rs index 276597ed..cb083f95 100644 --- a/src/obilayeredmap/src/tests/map.rs +++ b/src/obilayeredmap/src/tests/map.rs @@ -125,7 +125,7 @@ fn presence_partition(seqs: &[&[u8]], n_genomes: usize) -> (tempfile::TempDir, L let mode = IndexMode::Exact; let layer0 = layer_dir(dir.path(), 0); write_unitigs_at(&layer0, seqs); - Layer::::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| { + TypedLayer::::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| { (kmer.raw().wrapping_add(g as u64)) % 2 == 0 }).unwrap(); PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap(); @@ -173,7 +173,7 @@ fn detect_layer_storage_implicit() { write_unitigs_at(&layer0, &[b"AAAACGT"]); // Mode-1 build: MPHF + evidence + `layer_meta.json`, no matrix at all — // must read back as `Presence`/`Implicit`, never a third "empty" state. - Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap(); + TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap(); PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap(); let map = LayeredMap::<()>::open(dir.path()).unwrap(); @@ -203,7 +203,7 @@ fn detect_layer_evidence_exact_vs_approx() { let dir = tempdir().unwrap(); let layer0 = layer_dir(dir.path(), 0); write_unitigs_at(&layer0, &[b"AAAACGT"]); - Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap(); + TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap(); PartitionMeta { n_layers: 1, mode: IndexMode::Exact }.save(dir.path()).unwrap(); let map = LayeredMap::<()>::open(dir.path()).unwrap(); assert_eq!(map.detect_layer_evidence(0).unwrap(), EvidenceKind::Exact); @@ -212,7 +212,7 @@ fn detect_layer_evidence_exact_vs_approx() { let dir2 = tempdir().unwrap(); let layer0b = layer_dir(dir2.path(), 0); write_unitigs_at(&layer0b, &[b"AAAACGT"]); - Layer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap(); + TypedLayer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap(); PartitionMeta { n_layers: 1, mode: approx }.save(dir2.path()).unwrap(); let map2 = LayeredMap::<()>::open(dir2.path()).unwrap(); assert_eq!(map2.detect_layer_evidence(0).unwrap(), EvidenceKind::Approx);