Rename Layer<D> to TypedLayer and introduce heterogeneous Layer enum
Renames `Layer<D>` to `TypedLayer<D>` to establish a distinct typed abstraction. Introduces a new heterogeneous `Layer` enum that unifies count and presence storage with runtime dispatch, delegating operations to the underlying typed variants. Updates cache, index, and phylo consumers to align with the renamed type and new extension traits, preserving full test suite stability while preparing the foundation for multi-partition caching.
This commit is contained in:
Eric Coissac
@@ -1,18 +1,11 @@
|
||||
use rayon::prelude::*;
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::meta::IndexMode;
|
||||
use obilayeredmap::{Layer, OLMResult};
|
||||
use obilayeredmap::Layer;
|
||||
use obisys::progress_bar;
|
||||
|
||||
use obikindex::{KmerIndex, OKIResult};
|
||||
|
||||
use super::SiblingAnnex;
|
||||
use super::iter::SiblingLayerExt;
|
||||
|
||||
/// Every partition's already-open layers, built **once** for the whole
|
||||
/// `build_sibling_annex` run and shared (read-only) across every lookup, in
|
||||
/// every source layer, for the rest of the run — not reopened/re-mmap'd per
|
||||
@@ -31,119 +24,14 @@ use super::iter::SiblingLayerExt;
|
||||
/// partition for the entire run, regardless of how many source layers or
|
||||
/// lookups follow.
|
||||
///
|
||||
/// One `Layer<D>` per layer (MPHF + matrix bundled), not a separate
|
||||
/// `MphfLayer` and a separate `PersistentCompactIntMatrix`/
|
||||
/// `PersistentBitMatrix` in parallel arrays — `obilayeredmap::Layer` already
|
||||
/// *is* that bundle, with `find_slot` (MPHF-only, no data read),
|
||||
/// `n_cols`/`sub_matrix`/`fill_sub_matrix` (batched, sorted-internally
|
||||
/// column access) on top of it. Reinventing that pairing here would just be
|
||||
/// going back through the low-level pieces `Layer` already assembles.
|
||||
pub(super) enum Mat {
|
||||
Count(Layer<PersistentCompactIntMatrix>),
|
||||
/// Dense *or* sparse (`pack --sparse`d) presence — `PersistentBitMatrix`
|
||||
/// itself is a 4-way enum (`Columnar`/`Packed`/`Sparse`/`Implicit`) that
|
||||
/// already auto-detects sparse storage in its own `open()` (checks
|
||||
/// `presence/sparse_meta.json`) and dispatches every method
|
||||
/// (`row`/`fill_row`/`nonzero_iter`/...) across all four internally —
|
||||
/// see `DevDocMD/implementation/partition_layer_cache.md`. A separate
|
||||
/// `SparsePresence(Layer<PersistentSparseBitMatrix>)` arm used to exist
|
||||
/// here, opened via its own `presence/is_multi.prsb` check; it
|
||||
/// predated `PersistentBitMatrix` growing native sparse support and
|
||||
/// was pure duplication by the time it was removed — every method
|
||||
/// below dispatched it identically to this arm.
|
||||
Presence(Layer<PersistentBitMatrix>),
|
||||
}
|
||||
|
||||
impl Mat {
|
||||
/// Open one layer's matrix, auto-detecting count vs. presence from what
|
||||
/// is actually on disk — the single source of truth for *every* caller
|
||||
/// that opens a layer's own matrix (this module's cross-partition
|
||||
/// [`PartitionCache::build`] and `family_scan::scan_layer_families`'s
|
||||
/// own-layer lookup alike), so the two can never disagree. Sparse vs.
|
||||
/// dense presence storage is `PersistentBitMatrix::open`'s own concern
|
||||
/// (see the [`Presence`](Mat::Presence) variant's docs), not decided
|
||||
/// here.
|
||||
pub(super) fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult<Self> {
|
||||
if with_counts && layer_dir.join("counts").exists() {
|
||||
return Layer::<PersistentCompactIntMatrix>::open(layer_dir, mode).map(Mat::Count);
|
||||
}
|
||||
Layer::<PersistentBitMatrix>::open(layer_dir, mode).map(Mat::Presence)
|
||||
}
|
||||
|
||||
fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
|
||||
match self {
|
||||
Mat::Count(l) => l.find_slot(kmer),
|
||||
Mat::Presence(l) => l.find_slot(kmer),
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw MPHF batch lookup: kmer → slot, no membership check — for
|
||||
/// callers that already know every kmer is a member of *this* layer
|
||||
/// (e.g. it came from this layer's own `iter_minorants_batch`), so the
|
||||
/// evidence check `find_slot`/`find` would perform is redundant work.
|
||||
/// See `DevDocMD/architecture/siblings.md`: iteration-pipeline kmers use
|
||||
/// `index`, never `find`.
|
||||
pub(super) fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
|
||||
match self {
|
||||
Mat::Count(l) => l.index_batch(kmers),
|
||||
Mat::Presence(l) => l.index_batch(kmers),
|
||||
}
|
||||
}
|
||||
|
||||
/// This layer's own `SiblingLayerExt::iter_minorants_batch` — dispatch
|
||||
/// only, both arms return the same concrete `MinorantBatchIter` (it
|
||||
/// doesn't depend on `D`), so no boxing is needed.
|
||||
pub(super) fn iter_minorants_batch(
|
||||
&self,
|
||||
annex: std::sync::Arc<SiblingAnnex>,
|
||||
batch_size: usize,
|
||||
) -> super::iter::MinorantBatchIter {
|
||||
match self {
|
||||
Mat::Count(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
Mat::Presence(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn n_cols(&self) -> usize {
|
||||
match self {
|
||||
Mat::Count(l) => l.n_cols(),
|
||||
Mat::Presence(l) => l.n_cols(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` =
|
||||
/// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must
|
||||
/// have one entry per genome column, each resized to `slots.len()`.
|
||||
///
|
||||
/// Delegates entirely to `Layer<D>::fill_sub_matrix`, which sorts
|
||||
/// `slots` once internally for a sequential mmap sweep per column, then
|
||||
/// restores the original order — the same discipline this call site
|
||||
/// (and `find_presence_batch`) used to hand-roll with its own sort +
|
||||
/// genome-major loop. `Count` still needs one intermediate
|
||||
/// `Vec<Vec<u32>>` fetch (the underlying store only has an int
|
||||
/// sub-matrix, not a bool one), converted to presence (`!= 0`) in place
|
||||
/// — the sort/sequential-access win is unaffected, just one extra
|
||||
/// allocation pass over already-in-hand data.
|
||||
pub(super) fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec<bool>]) {
|
||||
match self {
|
||||
Mat::Presence(l) => l.fill_sub_matrix(slots, out),
|
||||
Mat::Count(l) => {
|
||||
let mut counts: Vec<Vec<u32>> = out.iter().map(|_| Vec::new()).collect();
|
||||
l.fill_sub_matrix(slots, &mut counts);
|
||||
for (o, c) in out.iter_mut().zip(counts.iter()) {
|
||||
o.clear();
|
||||
o.extend(c.iter().map(|&v| v != 0));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `mats[partition][layer]` holds `obilayeredmap::Layer` — the
|
||||
/// format-erased `Count`/`Presence` handle, not a sibling-specific type:
|
||||
/// this module used to bundle its own `Mat` enum here, duplicating exactly
|
||||
/// what `Layer` now does one crate down (see
|
||||
/// `DevDocMD/implementation/partition_layer_cache.md`). Its
|
||||
/// sibling-specific extension (`iter_minorants_batch`) lives in `iter.rs`.
|
||||
pub(super) struct PartitionCache {
|
||||
/// `mats[partition][layer]` = that partition's opened layers. Used by
|
||||
/// both [`obikindex::KmerIndex::build_sibling_annex`] and
|
||||
/// [`obikindex::KmerIndex::sibling_annex_stats`].
|
||||
mats: Vec<Vec<Mat>>,
|
||||
mats: Vec<Vec<Layer>>,
|
||||
/// Whether every `FamilyMask` field in this index's sibling annexes can
|
||||
/// be trusted as a real layer index (`n_layers <= 7`, the same decision
|
||||
/// `build_layer_sibling_annex` makes when writing them) — computed once
|
||||
@@ -163,9 +51,9 @@ impl PartitionCache {
|
||||
with_counts: bool,
|
||||
) -> OKIResult<Self> {
|
||||
let pb = progress_bar("open_partitions", n_parts as u64, "partitions");
|
||||
let built: Vec<(Vec<Mat>, usize)> = (0..n_parts)
|
||||
let built: Vec<(Vec<Layer>, usize)> = (0..n_parts)
|
||||
.into_par_iter()
|
||||
.map(|part| -> OKIResult<(Vec<Mat>, usize)> {
|
||||
.map(|part| -> OKIResult<(Vec<Layer>, usize)> {
|
||||
let index_dir = index.index_dir(part);
|
||||
if !index_dir.exists() {
|
||||
pb.inc(1);
|
||||
@@ -174,7 +62,7 @@ impl PartitionCache {
|
||||
let meta = index.partition_meta(part)?;
|
||||
let mut mats = Vec::with_capacity(meta.n_layers);
|
||||
for l in 0..meta.n_layers {
|
||||
let Ok(mat) = Mat::open(&index.layer_dir(part, l), &meta.mode, with_counts)
|
||||
let Ok(mat) = Layer::open(&index.layer_dir(part, l), &meta.mode, with_counts)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
@@ -331,7 +219,7 @@ impl PartitionCache {
|
||||
/// `hits` gets its slots (evidence-probed vs. trusted `index_batch`) — this
|
||||
/// is everything after that.
|
||||
fn resolve_layer_hits(
|
||||
mat: &Mat,
|
||||
mat: &Layer,
|
||||
hits: &[(usize, usize, u8)],
|
||||
n_genomes: usize,
|
||||
on_hit: &mut impl FnMut(usize, u8, usize),
|
||||
|
||||
@@ -60,9 +60,11 @@ use obipipeline::{ThrottleGuard, throttle};
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
use obikindex::KmerIndex;
|
||||
|
||||
use super::cache::{Mat, PartitionCache};
|
||||
use obilayeredmap::Layer;
|
||||
|
||||
use super::cache::PartitionCache;
|
||||
use super::helpers::central_base;
|
||||
use super::iter::SiblingEntry;
|
||||
use super::iter::{SiblingEntry, SiblingLayerExt};
|
||||
use super::{olm_to_ok, FamilyMask, SiblingAnnex, ANNEX_FILE_NAME};
|
||||
|
||||
/// Families per batch — see the module docs for the memory-vs-per-partition-
|
||||
@@ -129,10 +131,10 @@ pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult<Vec<PathBuf>> {
|
||||
/// generating this layer's batches — opened once, not per batch. No `cache`
|
||||
/// here: generation never touches the cross-partition cache, only this
|
||||
/// layer's own already-open matrix. No separate MPHF/`slot_kmer` either —
|
||||
/// `mat` (a `Layer<D>`) already bundles the MPHF, and each `SiblingEntry`
|
||||
/// `mat` (a `TypedLayer<D>`) already bundles the MPHF, and each `SiblingEntry`
|
||||
/// arrives with its kmer and mask already in hand from `iter_minorants_batch`.
|
||||
struct LayerCtx {
|
||||
mat: Mat,
|
||||
mat: Layer,
|
||||
n_parts: usize,
|
||||
n_genomes: usize,
|
||||
n_cols: usize,
|
||||
@@ -193,7 +195,7 @@ pub(super) fn scan_layer_families(
|
||||
let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?;
|
||||
let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?);
|
||||
|
||||
let mat = Mat::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?;
|
||||
let mat = Layer::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?;
|
||||
let n_cols = mat.n_cols().min(n_genomes);
|
||||
|
||||
let ctx = Arc::new(LayerCtx { mat, n_parts, n_genomes, n_cols, k });
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! Phylo/sibling-domain iteration over a layer — an extension trait, not a
|
||||
//! new field on `MphfLayer`/`Layer<D>`: "family"/"minorant" are phylo
|
||||
//! new field on `MphfLayer`/`TypedLayer<D>`: "family"/"minorant" are phylo
|
||||
//! concepts, `obilayeredmap` stays kmer/slot-mapping only (see
|
||||
//! `DevDocMD/architecture/siblings.md`).
|
||||
//!
|
||||
@@ -29,7 +29,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::{KmerIter, Layer, LayerData};
|
||||
use obilayeredmap::{KmerIter, TypedLayer, LayerData};
|
||||
|
||||
use super::{FamilyMask, SiblingAnnex};
|
||||
|
||||
@@ -128,10 +128,10 @@ fn collect_batch<I: Iterator>(inner: &mut I, batch_size: usize) -> Option<Vec<I:
|
||||
if batch.is_empty() { None } else { Some(batch) }
|
||||
}
|
||||
|
||||
/// Adds phylo/sibling iteration to any `Layer<D>` — the extension that
|
||||
/// Adds phylo/sibling iteration to any `TypedLayer<D>` — the extension that
|
||||
/// turns a plain layer into a "sibling layer". Generic over `D`
|
||||
/// (`LayerData`) rather than implemented once per matrix kind: kmer
|
||||
/// iteration doesn't depend on the data payload, and `Layer<D>` already
|
||||
/// iteration doesn't depend on the data payload, and `TypedLayer<D>` already
|
||||
/// delegates `iter_kmers`/`index`/`index_batch` to its inner MPHF for every
|
||||
/// `D` — reusing that instead of going back through a separate `MphfLayer`.
|
||||
pub trait SiblingLayerExt {
|
||||
@@ -154,7 +154,7 @@ pub trait SiblingLayerExt {
|
||||
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter;
|
||||
}
|
||||
|
||||
impl<D: LayerData> SiblingLayerExt for Layer<D> {
|
||||
impl<D: LayerData> SiblingLayerExt for TypedLayer<D> {
|
||||
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
|
||||
SiblingIter { kmers: self.iter_kmers(), annex, order: 0 }
|
||||
}
|
||||
@@ -171,3 +171,39 @@ impl<D: LayerData> SiblingLayerExt for Layer<D> {
|
||||
MinorantBatchIter { inner: self.iter_minorants(annex), batch_size }
|
||||
}
|
||||
}
|
||||
|
||||
/// Same extension, over `obilayeredmap::Layer` (the format-erased
|
||||
/// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch
|
||||
/// only, both arms return the same concrete iterator types (they don't
|
||||
/// depend on which `D` is inside), so no boxing is needed. `obilayeredmap`
|
||||
/// itself can't implement this: "family"/"minorant" are phylo concepts, it
|
||||
/// stays kmer/slot-mapping only (see the module docs above).
|
||||
impl SiblingLayerExt for obilayeredmap::Layer {
|
||||
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_siblings(annex),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_siblings(annex),
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_siblings_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> SiblingBatchIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size),
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_minorants(&self, annex: Arc<SiblingAnnex>) -> MinorantIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_minorants(annex),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_minorants(annex),
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,9 +156,9 @@ fn sibling_annex_works_after_pack_sparse() {
|
||||
// `pack --sparse` (`KmerIndex::pack_matrices(true)`) run on the merged
|
||||
// index before building the sibling annex — proves `PartitionCache`'s
|
||||
// sparse-detection (`Mat::SparsePresence`, gated on `presence/
|
||||
// is_multi.prsb`) and the generic `Layer<D>` methods it relies on
|
||||
// is_multi.prsb`) and the generic `TypedLayer<D>` methods it relies on
|
||||
// actually round-trip through the real build pipeline, not just the
|
||||
// unit-level `Layer<PersistentSparseBitMatrix>` tests in
|
||||
// unit-level `TypedLayer<PersistentSparseBitMatrix>` tests in
|
||||
// `obilayeredmap`.
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
|
||||
Reference in New Issue
Block a user