Push zunrplorkwkt #70

Merged
coissac merged 93 commits from push-zunrplorkwkt into main 2026-08-28 23:15:38 +00:00
15 changed files with 259 additions and 197 deletions
Showing only changes of commit 1c54e60c9a - Show all commits
@@ -133,14 +133,35 @@ needed — `obikindex → obikpartitionner` stays the same direction as before.
Full workspace test suite green (0 failed) after, including all 27
`obikphylo::siblings` tests.
Still open: (1)/(2) themselves — `AnyLayer` (or whatever it ends up named;
`AnyLayer` was rejected as a placeholder, no replacement chosen yet) and
`KmerPartition` (singular, one partition's open layers) are not built yet.
`obikphylo::siblings::cache::{Mat, PartitionCache}` and
`obikpartitionner::query_layer` (now `obikindex::query_layer`)'s
`QueryLayer` still each independently bundle MPHF+matrix — unchanged by
this restructuring, which was purely about *where* code lives, not about
building the heterogeneous-layer cache itself.
## (1) done (2026-08-20): `Layer` is now the heterogeneous handle, `Mat` is gone
Resolved the naming question left open above. `Layer<D>` (the old
generic/monomorphic type) renamed to `TypedLayer<D>` throughout
(`obilayeredmap`, `obikindex`, `obikphylo` — 12 files, mechanical) to free
`Layer` for the type that's actually meant to be everyone's default
handle. `obilayeredmap::content_layer::Layer` (re-exported at the crate
root) is that type — `Count(TypedLayer<PersistentCompactIntMatrix>)`/
`Presence(TypedLayer<PersistentBitMatrix>)`, `Layer::open` doing the same
disk probe `Mat::open` used to, `find_slot`/`index_batch`/`n_cols`/
`fill_sub_matrix_carries` dispatching per variant exactly as `Mat` did.
`obikphylo::siblings::cache::Mat` deleted outright — `PartitionCache` now
holds `Vec<Vec<obilayeredmap::Layer>>` directly. The one sibling-specific
method `Mat` carried (`iter_minorants_batch`) is not on `obilayeredmap::
Layer` (phylo concepts don't belong in `obilayeredmap`) — it's an
`impl SiblingLayerExt for obilayeredmap::Layer` in `iter.rs`, dispatching
to each variant's existing `impl<D: LayerData> SiblingLayerExt for
TypedLayer<D>`.
Full workspace suite green (0 failed) after, including all 27
`obikphylo::siblings` tests.
Still not built: (2) — `KmerPartition` (singular, one partition's open
`Vec<Layer>`) and a multi-partition cache in `obikpartitionner` to replace
`obikphylo::siblings::cache::PartitionCache` and `obikindex::query_layer`'s
still-separate `QueryLayer` (which still independently bundles MPHF+matrix,
2-way not using `Layer` at all). Both remaining consumers now sit one
`Layer::open` call away from unifying onto (2) once it exists.
## The problem
+4 -4
View File
@@ -10,7 +10,7 @@ use obipipeline::{
use obidebruinj::GraphDeBruijn;
use obikseq::CanonicalKmer;
use obilayeredmap::{IndexMode, Layer};
use obilayeredmap::{IndexMode, TypedLayer};
use obiskio::{SKError, SKResult};
use crate::common::olm_to_sk;
@@ -125,7 +125,7 @@ pub(crate) fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> SKRe
let n_kmers = g.len();
g.compute_degrees_and_mark_starts();
std::fs::create_dir_all(layer_dir)?;
let mut uw = Layer::<()>::unitig_writer(layer_dir).map_err(|e| olm_to_sk(e, "graph pipeline"))?;
let mut uw = TypedLayer::<()>::unitig_writer(layer_dir).map_err(|e| olm_to_sk(e, "graph pipeline"))?;
g.try_for_each_unitig(|unitig| uw.write(unitig))?;
uw.close()?;
drop(g);
@@ -134,7 +134,7 @@ pub(crate) fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> SKRe
// ── materialize_layer ─────────────────────────────────────────────────────────
/// Phase 2 (full): write_graph_as_unitigs + `Layer::<()>::build`.
/// Phase 2 (full): write_graph_as_unitigs + `TypedLayer::<()>::build`.
///
/// Returns n_kmers.
pub(crate) fn materialize_layer(
@@ -145,7 +145,7 @@ pub(crate) fn materialize_layer(
) -> SKResult<usize> {
let n = write_graph_as_unitigs(g, layer_dir)?;
debug!("materialize_layer: unitigs written ({n} kmers), building MPHF");
Layer::<()>::build(layer_dir, block_bits, evidence)
TypedLayer::<()>::build(layer_dir, block_bits, evidence)
.map_err(|e| olm_to_sk(e, "graph pipeline"))?;
debug!("materialize_layer: MPHF build done");
Ok(n)
+2 -2
View File
@@ -6,7 +6,7 @@ use epserde::prelude::*;
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn;
use obilayeredmap::meta::PartitionMeta;
use obilayeredmap::{IndexMode, layer::Layer};
use obilayeredmap::{IndexMode, layer::TypedLayer};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
@@ -96,7 +96,7 @@ impl KmerIndex {
let n_kmers =
if with_counts {
let n = write_graph_as_unitigs(g, &layer_dir)?;
Layer::<PersistentCompactIntMatrix>::build(&layer_dir, block_bits, mode, |kmer| {
TypedLayer::<PersistentCompactIntMatrix>::build(&layer_dir, block_bits, mode, |kmer| {
match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1,
+2 -2
View File
@@ -20,7 +20,7 @@ use tracing::debug;
use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder};
use obikseq::CanonicalKmer;
use obilayeredmap::{IndexMode, Layer, LayeredMap, MphfOnly, layer_dir};
use obilayeredmap::{IndexMode, TypedLayer, LayeredMap, MphfOnly, layer_dir};
use obiskio::{SKError, SKResult, UnitigFileReader};
use crate::common::{ColBuilder, load_meta, olm_to_sk};
@@ -232,7 +232,7 @@ impl KmerIndex {
// (all slots true — every kmer in those layers belongs to genome_0).
if n_dst_genomes == 1 && mode == MergeMode::Presence {
for l in 0..n_dst_layers {
Layer::<()>::init_presence_matrix(
TypedLayer::<()>::init_presence_matrix(
&layer_dir(&dst_index_dir, l),
dst_map.layer(l).n(),
)
+3 -3
View File
@@ -1,4 +1,4 @@
use obilayeredmap::{IndexMode, layer::Layer};
use obilayeredmap::{IndexMode, layer::TypedLayer};
use obisys::{Reporter, Stage, progress_bar};
use std::fs;
use std::path::Path;
@@ -90,10 +90,10 @@ fn reindex_partition(
fn reindex_layer(layer_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
match target {
IndexMode::Exact => {
Layer::<()>::build_exact_evidence(layer_dir, block_bits).map_err(olm_to_oki)?;
TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits).map_err(olm_to_oki)?;
}
IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => {
Layer::<()>::build_approx_evidence(layer_dir, *b, *z).map_err(olm_to_oki)?;
TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z).map_err(olm_to_oki)?;
}
}
remove_stale_evidence(layer_dir, target)
+12 -124
View File
@@ -1,18 +1,11 @@
use rayon::prelude::*;
use std::path::Path;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use obilayeredmap::meta::IndexMode;
use obilayeredmap::{Layer, OLMResult};
use obilayeredmap::Layer;
use obisys::progress_bar;
use obikindex::{KmerIndex, OKIResult};
use super::SiblingAnnex;
use super::iter::SiblingLayerExt;
/// Every partition's already-open layers, built **once** for the whole
/// `build_sibling_annex` run and shared (read-only) across every lookup, in
/// every source layer, for the rest of the run — not reopened/re-mmap'd per
@@ -31,119 +24,14 @@ use super::iter::SiblingLayerExt;
/// partition for the entire run, regardless of how many source layers or
/// lookups follow.
///
/// One `Layer<D>` per layer (MPHF + matrix bundled), not a separate
/// `MphfLayer` and a separate `PersistentCompactIntMatrix`/
/// `PersistentBitMatrix` in parallel arrays — `obilayeredmap::Layer` already
/// *is* that bundle, with `find_slot` (MPHF-only, no data read),
/// `n_cols`/`sub_matrix`/`fill_sub_matrix` (batched, sorted-internally
/// column access) on top of it. Reinventing that pairing here would just be
/// going back through the low-level pieces `Layer` already assembles.
pub(super) enum Mat {
Count(Layer<PersistentCompactIntMatrix>),
/// Dense *or* sparse (`pack --sparse`d) presence — `PersistentBitMatrix`
/// itself is a 4-way enum (`Columnar`/`Packed`/`Sparse`/`Implicit`) that
/// already auto-detects sparse storage in its own `open()` (checks
/// `presence/sparse_meta.json`) and dispatches every method
/// (`row`/`fill_row`/`nonzero_iter`/...) across all four internally —
/// see `DevDocMD/implementation/partition_layer_cache.md`. A separate
/// `SparsePresence(Layer<PersistentSparseBitMatrix>)` arm used to exist
/// here, opened via its own `presence/is_multi.prsb` check; it
/// predated `PersistentBitMatrix` growing native sparse support and
/// was pure duplication by the time it was removed — every method
/// below dispatched it identically to this arm.
Presence(Layer<PersistentBitMatrix>),
}
impl Mat {
/// Open one layer's matrix, auto-detecting count vs. presence from what
/// is actually on disk — the single source of truth for *every* caller
/// that opens a layer's own matrix (this module's cross-partition
/// [`PartitionCache::build`] and `family_scan::scan_layer_families`'s
/// own-layer lookup alike), so the two can never disagree. Sparse vs.
/// dense presence storage is `PersistentBitMatrix::open`'s own concern
/// (see the [`Presence`](Mat::Presence) variant's docs), not decided
/// here.
pub(super) fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult<Self> {
if with_counts && layer_dir.join("counts").exists() {
return Layer::<PersistentCompactIntMatrix>::open(layer_dir, mode).map(Mat::Count);
}
Layer::<PersistentBitMatrix>::open(layer_dir, mode).map(Mat::Presence)
}
fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
match self {
Mat::Count(l) => l.find_slot(kmer),
Mat::Presence(l) => l.find_slot(kmer),
}
}
/// Raw MPHF batch lookup: kmer → slot, no membership check — for
/// callers that already know every kmer is a member of *this* layer
/// (e.g. it came from this layer's own `iter_minorants_batch`), so the
/// evidence check `find_slot`/`find` would perform is redundant work.
/// See `DevDocMD/architecture/siblings.md`: iteration-pipeline kmers use
/// `index`, never `find`.
pub(super) fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
match self {
Mat::Count(l) => l.index_batch(kmers),
Mat::Presence(l) => l.index_batch(kmers),
}
}
/// This layer's own `SiblingLayerExt::iter_minorants_batch` — dispatch
/// only, both arms return the same concrete `MinorantBatchIter` (it
/// doesn't depend on `D`), so no boxing is needed.
pub(super) fn iter_minorants_batch(
&self,
annex: std::sync::Arc<SiblingAnnex>,
batch_size: usize,
) -> super::iter::MinorantBatchIter {
match self {
Mat::Count(l) => l.iter_minorants_batch(annex, batch_size),
Mat::Presence(l) => l.iter_minorants_batch(annex, batch_size),
}
}
pub(super) fn n_cols(&self) -> usize {
match self {
Mat::Count(l) => l.n_cols(),
Mat::Presence(l) => l.n_cols(),
}
}
/// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` =
/// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must
/// have one entry per genome column, each resized to `slots.len()`.
///
/// Delegates entirely to `Layer<D>::fill_sub_matrix`, which sorts
/// `slots` once internally for a sequential mmap sweep per column, then
/// restores the original order — the same discipline this call site
/// (and `find_presence_batch`) used to hand-roll with its own sort +
/// genome-major loop. `Count` still needs one intermediate
/// `Vec<Vec<u32>>` fetch (the underlying store only has an int
/// sub-matrix, not a bool one), converted to presence (`!= 0`) in place
/// — the sort/sequential-access win is unaffected, just one extra
/// allocation pass over already-in-hand data.
pub(super) fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec<bool>]) {
match self {
Mat::Presence(l) => l.fill_sub_matrix(slots, out),
Mat::Count(l) => {
let mut counts: Vec<Vec<u32>> = out.iter().map(|_| Vec::new()).collect();
l.fill_sub_matrix(slots, &mut counts);
for (o, c) in out.iter_mut().zip(counts.iter()) {
o.clear();
o.extend(c.iter().map(|&v| v != 0));
}
}
}
}
}
/// `mats[partition][layer]` holds `obilayeredmap::Layer` — the
/// format-erased `Count`/`Presence` handle, not a sibling-specific type:
/// this module used to bundle its own `Mat` enum here, duplicating exactly
/// what `Layer` now does one crate down (see
/// `DevDocMD/implementation/partition_layer_cache.md`). Its
/// sibling-specific extension (`iter_minorants_batch`) lives in `iter.rs`.
pub(super) struct PartitionCache {
/// `mats[partition][layer]` = that partition's opened layers. Used by
/// both [`obikindex::KmerIndex::build_sibling_annex`] and
/// [`obikindex::KmerIndex::sibling_annex_stats`].
mats: Vec<Vec<Mat>>,
mats: Vec<Vec<Layer>>,
/// Whether every `FamilyMask` field in this index's sibling annexes can
/// be trusted as a real layer index (`n_layers <= 7`, the same decision
/// `build_layer_sibling_annex` makes when writing them) — computed once
@@ -163,9 +51,9 @@ impl PartitionCache {
with_counts: bool,
) -> OKIResult<Self> {
let pb = progress_bar("open_partitions", n_parts as u64, "partitions");
let built: Vec<(Vec<Mat>, usize)> = (0..n_parts)
let built: Vec<(Vec<Layer>, usize)> = (0..n_parts)
.into_par_iter()
.map(|part| -> OKIResult<(Vec<Mat>, usize)> {
.map(|part| -> OKIResult<(Vec<Layer>, usize)> {
let index_dir = index.index_dir(part);
if !index_dir.exists() {
pb.inc(1);
@@ -174,7 +62,7 @@ impl PartitionCache {
let meta = index.partition_meta(part)?;
let mut mats = Vec::with_capacity(meta.n_layers);
for l in 0..meta.n_layers {
let Ok(mat) = Mat::open(&index.layer_dir(part, l), &meta.mode, with_counts)
let Ok(mat) = Layer::open(&index.layer_dir(part, l), &meta.mode, with_counts)
else {
continue;
};
@@ -331,7 +219,7 @@ impl PartitionCache {
/// `hits` gets its slots (evidence-probed vs. trusted `index_batch`) — this
/// is everything after that.
fn resolve_layer_hits(
mat: &Mat,
mat: &Layer,
hits: &[(usize, usize, u8)],
n_genomes: usize,
on_hit: &mut impl FnMut(usize, u8, usize),
+7 -5
View File
@@ -60,9 +60,11 @@ use obipipeline::{ThrottleGuard, throttle};
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use super::cache::{Mat, PartitionCache};
use obilayeredmap::Layer;
use super::cache::PartitionCache;
use super::helpers::central_base;
use super::iter::SiblingEntry;
use super::iter::{SiblingEntry, SiblingLayerExt};
use super::{olm_to_ok, FamilyMask, SiblingAnnex, ANNEX_FILE_NAME};
/// Families per batch — see the module docs for the memory-vs-per-partition-
@@ -129,10 +131,10 @@ pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult<Vec<PathBuf>> {
/// generating this layer's batches — opened once, not per batch. No `cache`
/// here: generation never touches the cross-partition cache, only this
/// layer's own already-open matrix. No separate MPHF/`slot_kmer` either —
/// `mat` (a `Layer<D>`) already bundles the MPHF, and each `SiblingEntry`
/// `mat` (a `TypedLayer<D>`) already bundles the MPHF, and each `SiblingEntry`
/// arrives with its kmer and mask already in hand from `iter_minorants_batch`.
struct LayerCtx {
mat: Mat,
mat: Layer,
n_parts: usize,
n_genomes: usize,
n_cols: usize,
@@ -193,7 +195,7 @@ pub(super) fn scan_layer_families(
let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?;
let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?);
let mat = Mat::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?;
let mat = Layer::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?;
let n_cols = mat.n_cols().min(n_genomes);
let ctx = Arc::new(LayerCtx { mat, n_parts, n_genomes, n_cols, k });
+41 -5
View File
@@ -1,5 +1,5 @@
//! Phylo/sibling-domain iteration over a layer — an extension trait, not a
//! new field on `MphfLayer`/`Layer<D>`: "family"/"minorant" are phylo
//! new field on `MphfLayer`/`TypedLayer<D>`: "family"/"minorant" are phylo
//! concepts, `obilayeredmap` stays kmer/slot-mapping only (see
//! `DevDocMD/architecture/siblings.md`).
//!
@@ -29,7 +29,7 @@
use std::sync::Arc;
use obikseq::CanonicalKmer;
use obilayeredmap::{KmerIter, Layer, LayerData};
use obilayeredmap::{KmerIter, TypedLayer, LayerData};
use super::{FamilyMask, SiblingAnnex};
@@ -128,10 +128,10 @@ fn collect_batch<I: Iterator>(inner: &mut I, batch_size: usize) -> Option<Vec<I:
if batch.is_empty() { None } else { Some(batch) }
}
/// Adds phylo/sibling iteration to any `Layer<D>` — the extension that
/// Adds phylo/sibling iteration to any `TypedLayer<D>` — the extension that
/// turns a plain layer into a "sibling layer". Generic over `D`
/// (`LayerData`) rather than implemented once per matrix kind: kmer
/// iteration doesn't depend on the data payload, and `Layer<D>` already
/// iteration doesn't depend on the data payload, and `TypedLayer<D>` already
/// delegates `iter_kmers`/`index`/`index_batch` to its inner MPHF for every
/// `D` — reusing that instead of going back through a separate `MphfLayer`.
pub trait SiblingLayerExt {
@@ -154,7 +154,7 @@ pub trait SiblingLayerExt {
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter;
}
impl<D: LayerData> SiblingLayerExt for Layer<D> {
impl<D: LayerData> SiblingLayerExt for TypedLayer<D> {
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
SiblingIter { kmers: self.iter_kmers(), annex, order: 0 }
}
@@ -171,3 +171,39 @@ impl<D: LayerData> SiblingLayerExt for Layer<D> {
MinorantBatchIter { inner: self.iter_minorants(annex), batch_size }
}
}
/// Same extension, over `obilayeredmap::Layer` (the format-erased
/// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch
/// only, both arms return the same concrete iterator types (they don't
/// depend on which `D` is inside), so no boxing is needed. `obilayeredmap`
/// itself can't implement this: "family"/"minorant" are phylo concepts, it
/// stays kmer/slot-mapping only (see the module docs above).
impl SiblingLayerExt for obilayeredmap::Layer {
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_siblings(annex),
obilayeredmap::Layer::Presence(l) => l.iter_siblings(annex),
}
}
fn iter_siblings_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> SiblingBatchIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size),
obilayeredmap::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size),
}
}
fn iter_minorants(&self, annex: Arc<SiblingAnnex>) -> MinorantIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_minorants(annex),
obilayeredmap::Layer::Presence(l) => l.iter_minorants(annex),
}
}
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size),
obilayeredmap::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size),
}
}
}
+2 -2
View File
@@ -156,9 +156,9 @@ fn sibling_annex_works_after_pack_sparse() {
// `pack --sparse` (`KmerIndex::pack_matrices(true)`) run on the merged
// index before building the sibling annex — proves `PartitionCache`'s
// sparse-detection (`Mat::SparsePresence`, gated on `presence/
// is_multi.prsb`) and the generic `Layer<D>` methods it relies on
// is_multi.prsb`) and the generic `TypedLayer<D>` methods it relies on
// actually round-trip through the real build pipeline, not just the
// unit-level `Layer<PersistentSparseBitMatrix>` tests in
// unit-level `TypedLayer<PersistentSparseBitMatrix>` tests in
// `obilayeredmap`.
let dir = tempdir().unwrap();
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
+113
View File
@@ -0,0 +1,113 @@
//! [`Layer`] — the format-erased layer handle every caller outside this
//! crate should reach for. `TypedLayer<D>` (`layer.rs`) is monomorphic: a
//! `Vec<TypedLayer<D>>` needs one `D` fixed at compile time for every
//! element, which real partitions don't respect (`with_counts` can differ
//! layer to layer across merges; see
//! `DevDocMD/implementation/partition_layer_cache.md`). `Layer` is the
//! sum type that lets a partition's layers be held together regardless —
//! `Count`/`Presence` each wrap the one concrete `TypedLayer<D>` that
//! content implies, chosen by [`Layer::open`]'s own disk probe.
//!
//! This is exactly the shape `obikphylo::siblings::cache::Mat` used to
//! reimplement locally, minus its one sibling-specific method
//! (`iter_minorants_batch`, which stays an extension trait over there —
//! `obilayeredmap` has no business knowing about sibling annexes).
use std::path::Path;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use crate::error::OLMResult;
use crate::layer::{LayerContent, TypedLayer, COUNTS_DIR};
use crate::meta::IndexMode;
use crate::mphf_layer::EvidenceKind;
/// One layer, its content (`Count`/`Presence`) resolved at open time rather
/// than at compile time — see the module docs.
pub enum Layer {
Count(TypedLayer<PersistentCompactIntMatrix>),
Presence(TypedLayer<PersistentBitMatrix>),
}
impl Layer {
/// Open one layer, auto-detecting count vs. presence from what is
/// actually on disk (`counts/` present and wanted, else presence) — the
/// single source of truth every caller that opens a layer's own matrix
/// should share, so two callers can never disagree about which format a
/// given layer is. Dense vs. sparse presence storage is
/// `PersistentBitMatrix::open`'s own concern (it's a 4-way
/// `Columnar`/`Packed`/`Sparse`/`Implicit` enum internally), not decided
/// here.
pub fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult<Self> {
if with_counts && layer_dir.join(COUNTS_DIR).exists() {
return TypedLayer::<PersistentCompactIntMatrix>::open(layer_dir, mode).map(Layer::Count);
}
TypedLayer::<PersistentBitMatrix>::open(layer_dir, mode).map(Layer::Presence)
}
pub fn content(&self) -> LayerContent {
match self {
Layer::Count(_) => LayerContent::Count,
Layer::Presence(_) => LayerContent::Presence,
}
}
pub fn evidence_kind(&self) -> EvidenceKind {
match self {
Layer::Count(l) => l.evidence_kind(),
Layer::Presence(l) => l.evidence_kind(),
}
}
pub fn n(&self) -> usize {
match self {
Layer::Count(l) => l.n(),
Layer::Presence(l) => l.n(),
}
}
pub fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
match self {
Layer::Count(l) => l.find_slot(kmer),
Layer::Presence(l) => l.find_slot(kmer),
}
}
/// Raw MPHF batch lookup: kmer → slot, no membership check — for
/// callers that already know every kmer is a member of *this* layer, so
/// the evidence check `find_slot`/`find` would perform is redundant.
pub fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
match self {
Layer::Count(l) => l.index_batch(kmers),
Layer::Presence(l) => l.index_batch(kmers),
}
}
pub fn n_cols(&self) -> usize {
match self {
Layer::Count(l) => l.n_cols(),
Layer::Presence(l) => l.n_cols(),
}
}
/// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` =
/// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must
/// have one entry per genome column, each resized to `slots.len()`.
/// `Count` needs one intermediate `Vec<Vec<u32>>` fetch (the underlying
/// store only has an int sub-matrix, not a bool one), converted to
/// presence (`!= 0`) in place.
pub fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec<bool>]) {
match self {
Layer::Presence(l) => l.fill_sub_matrix(slots, out),
Layer::Count(l) => {
let mut counts: Vec<Vec<u32>> = out.iter().map(|_| Vec::new()).collect();
l.fill_sub_matrix(slots, &mut counts);
for (o, c) in out.iter_mut().zip(counts.iter()) {
o.clear();
o.extend(c.iter().map(|&v| v != 0));
}
}
}
}
}
+16 -16
View File
@@ -27,7 +27,7 @@ pub trait LayerData: Sized {
fn read(&self, slot: usize) -> Self::Item;
}
/// Layer directory path for layer `i` within a partition's index root — the
/// TypedLayer directory path for layer `i` within a partition's index root — the
/// single source of truth for the on-disk `layer_N` naming convention
/// (mirrors what `LayeredMap::open`/`push_layer` already use internally).
///
@@ -43,7 +43,7 @@ pub fn layer_dir(root: &Path, i: usize) -> PathBuf {
/// Opens layer `i`'s data only, skipping the MPHF — for callers that only
/// need matrix-level operations (distance traits, column weights, group
/// filters, sub-matrix extraction) and never look up a kmer for this layer.
/// `Layer<D>::open` always pays for the MPHF too (and doesn't expose `data`
/// `TypedLayer<D>::open` always pays for the MPHF too (and doesn't expose `data`
/// once open), so it's the wrong tool for these; this is the other half of
/// the same `D::open(layer_dir)` call, without the MPHF alongside it.
///
@@ -89,7 +89,7 @@ impl LayerData for PersistentSparseBitMatrix {
/// What a layer's data matrix represents — orthogonal to *how* it's stored
/// on disk (`obicompactvec::StorageKind`'s concern). Only meaningful for
/// `D` that actually carry a matrix: `Layer<()>` (mode 1, set membership,
/// `D` that actually carry a matrix: `TypedLayer<()>` (mode 1, set membership,
/// no matrix at all) has neither a `LayerContent` nor a `content()` method
/// — it's a write-time-only state, never a queryable content. Once a layer
/// is closed, "no matrix file" reads back as `Presence` via
@@ -115,7 +115,7 @@ impl LayerContent {
}
/// Implemented only by `D` that carry a real matrix — gates
/// [`Layer::content`](Layer::content) so `Layer<()>` doesn't get it.
/// [`TypedLayer::content`](TypedLayer::content) so `TypedLayer<()>` doesn't get it.
pub trait HasLayerContent {
const CONTENT: LayerContent;
}
@@ -135,7 +135,7 @@ impl HasLayerContent for PersistentSparseBitMatrix {
// ── StorageKind passthrough ────────────────────────────────────────────────────
/// Implemented only by `D` that expose their own `storage_kind()` — gates
/// [`Layer::storage_kind`]. `PersistentSparseBitMatrix` has a single,
/// [`TypedLayer::storage_kind`]. `PersistentSparseBitMatrix` has a single,
/// fixed on-disk format (no `Columnar`/`Packed`/`Implicit` variants of its
/// own), so it's deliberately not given an impl: there's nothing to report
/// beyond what `content()` already says.
@@ -157,7 +157,7 @@ impl HasStorageKind for PersistentBitMatrix {
// ── Structures ────────────────────────────────────────────────────────────────
pub struct Layer<D: LayerData = ()> {
pub struct TypedLayer<D: LayerData = ()> {
mphf: MphfLayer,
data: D,
}
@@ -169,7 +169,7 @@ pub struct Hit<T = ()> {
// ── Common read path ──────────────────────────────────────────────────────────
impl<D: LayerData> Layer<D> {
impl<D: LayerData> TypedLayer<D> {
pub fn open(path: &Path, mode: &IndexMode) -> OLMResult<Self> {
let mphf = MphfLayer::open(path, mode)?;
let data = D::open(path)?;
@@ -249,7 +249,7 @@ impl<D: LayerData> Layer<D> {
}
}
impl<D: LayerData + HasLayerContent> Layer<D> {
impl<D: LayerData + HasLayerContent> TypedLayer<D> {
/// This already-open layer's content (`Count`/`Presence`) — a
/// compile-time fact about `D`, not a runtime read.
pub const fn content(&self) -> LayerContent {
@@ -257,7 +257,7 @@ impl<D: LayerData + HasLayerContent> Layer<D> {
}
}
impl<D: LayerData + HasStorageKind> Layer<D> {
impl<D: LayerData + HasStorageKind> TypedLayer<D> {
/// This already-open layer's storage format — delegates to `D`'s own
/// `storage_kind()`, reading a discriminant already in memory.
pub fn storage_kind(&self) -> obicompactvec::StorageKind {
@@ -267,7 +267,7 @@ impl<D: LayerData + HasStorageKind> Layer<D> {
// ── Mode 1 — set membership ───────────────────────────────────────────────────
impl Layer<()> {
impl TypedLayer<()> {
pub fn build(out_dir: &Path, block_bits: u8, mode: &IndexMode) -> OLMResult<usize> {
MphfLayer::build(out_dir, block_bits, mode, &mut |_, _| Ok(()))
}
@@ -284,7 +284,7 @@ impl Layer<()> {
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
impl Layer<PersistentCompactIntMatrix> {
impl TypedLayer<PersistentCompactIntMatrix> {
pub fn build(
out_dir: &Path,
block_bits: u8,
@@ -317,7 +317,7 @@ impl Layer<PersistentCompactIntMatrix> {
// ── Mode 2 — count matrix column append ──────────────────────────────────────
impl Layer<PersistentCompactIntMatrix> {
impl TypedLayer<PersistentCompactIntMatrix> {
pub fn append_genome_column(
layer_dir: &Path,
value_of: impl Fn(usize) -> u32,
@@ -357,14 +357,14 @@ impl Layer<PersistentCompactIntMatrix> {
//
// `n_cols`/`sub_matrix`/`fill_sub_matrix` are identical in shape for any
// `D` satisfying `BinaryMatrix` — genuinely generic (not just two
// near-identical impl blocks) so `Layer<PersistentSparseBitMatrix>` gets
// them for free, matching `Layer<PersistentBitMatrix>` at the same call
// near-identical impl blocks) so `TypedLayer<PersistentSparseBitMatrix>` gets
// them for free, matching `TypedLayer<PersistentBitMatrix>` at the same call
// sites (see `obikphylo::siblings::cache::Mat`). Construction
// (`append_genome_column`/`build_presence`) stays `PersistentBitMatrix`-only
// below: sparse matrices aren't built column-by-column, they're built
// row-by-row from an already-built dense layer
// (`PersistentSparseBitMatrixBuilder::build_from_dense`).
impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix> Layer<D> {
impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix> TypedLayer<D> {
/// Number of genome columns in this layer's presence matrix — see
/// `PersistentBitMatrix::n_cols`'s docs for the `Implicit` mono-genome
/// special case (always reports `1`, regardless of the index's real
@@ -395,7 +395,7 @@ impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix> Layer<D> {
}
}
impl Layer<PersistentBitMatrix> {
impl TypedLayer<PersistentBitMatrix> {
pub fn append_genome_column(
layer_dir: &Path,
value_of: impl Fn(usize) -> bool,
+3 -1
View File
@@ -1,3 +1,4 @@
pub mod content_layer;
pub mod error;
pub mod evidence;
pub mod fingerprint;
@@ -7,8 +8,9 @@ pub mod map;
pub mod meta;
pub(crate) mod mphf_layer;
pub use content_layer::Layer;
pub use error::{OLMError, OLMResult};
pub use layer::{layer_dir, open_data, HasLayerContent, HasStorageKind, Hit, Layer, LayerContent, LayerData};
pub use layer::{layer_dir, open_data, HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer};
pub use layered_store::LayeredStore;
pub use map::LayeredMap;
pub use meta::{IndexMode, PartitionMeta};
+12 -12
View File
@@ -7,7 +7,7 @@ use obikseq::CanonicalKmer;
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
use crate::error::{OLMError, OLMResult};
use crate::layer::{layer_dir, Hit, Layer, LayerContent, LayerData};
use crate::layer::{layer_dir, Hit, TypedLayer, LayerContent, LayerData};
use crate::meta::{IndexMode, PartitionMeta};
use crate::mphf_layer::EvidenceKind;
@@ -19,7 +19,7 @@ use crate::mphf_layer::EvidenceKind;
pub struct LayeredMap<D: LayerData = ()> {
root: PathBuf,
meta: PartitionMeta,
layers: Vec<Layer<D>>,
layers: Vec<TypedLayer<D>>,
}
// ── Common methods ────────────────────────────────────────────────────────────
@@ -30,7 +30,7 @@ impl<D: LayerData> LayeredMap<D> {
pub fn open(root: &Path) -> OLMResult<Self> {
let meta = PartitionMeta::load(root)?;
let layers = (0..meta.n_layers)
.map(|i| Layer::<D>::open(&layer_dir(root, i), &meta.mode))
.map(|i| TypedLayer::<D>::open(&layer_dir(root, i), &meta.mode))
.collect::<OLMResult<Vec<_>>>()?;
Ok(Self { root: root.to_owned(), meta, layers })
}
@@ -44,18 +44,18 @@ impl<D: LayerData> LayeredMap<D> {
}
pub fn n_layers(&self) -> usize { self.layers.len() }
pub fn layer(&self, i: usize) -> &Layer<D> { &self.layers[i] }
pub fn layer(&self, i: usize) -> &TypedLayer<D> { &self.layers[i] }
pub fn mode(&self) -> &IndexMode { &self.meta.mode }
/// Layer `i`'s content (`Count`/`Presence`) — a lightweight disk probe,
/// TypedLayer `i`'s content (`Count`/`Presence`) — a lightweight disk probe,
/// no MPHF/matrix opened. For picking a `D` before committing to
/// [`Layer::open`], or for reporting/diagnostics over an already-open
/// [`TypedLayer::open`], or for reporting/diagnostics over an already-open
/// `LayeredMap` without re-opening every layer under a different `D`.
pub fn detect_layer_content(&self, i: usize) -> LayerContent {
LayerContent::detect(&layer_dir(&self.root, i))
}
/// Layer `i`'s storage format — a lightweight disk probe, no
/// TypedLayer `i`'s storage format — a lightweight disk probe, no
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
pub fn detect_layer_storage(&self, i: usize) -> OLMResult<StorageKind> {
let dir = layer_dir(&self.root, i);
@@ -65,7 +65,7 @@ impl<D: LayerData> LayeredMap<D> {
}
}
/// Layer `i`'s evidence mode — a lightweight disk probe, no
/// TypedLayer `i`'s evidence mode — a lightweight disk probe, no
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
pub fn detect_layer_evidence(&self, i: usize) -> OLMResult<EvidenceKind> {
EvidenceKind::detect(&layer_dir(&self.root, i))
@@ -81,13 +81,13 @@ impl<D: LayerData> LayeredMap<D> {
pub fn next_layer_writer(&self) -> OLMResult<UnitigFileWriter> {
let dir = layer_dir(&self.root, self.layers.len());
Layer::<D>::unitig_writer(&dir)
TypedLayer::<D>::unitig_writer(&dir)
}
fn append_layer(&mut self) -> OLMResult<()> {
let i = self.layers.len();
let dir = layer_dir(&self.root, i);
self.layers.push(Layer::<D>::open(&dir, &self.meta.mode)?);
self.layers.push(TypedLayer::<D>::open(&dir, &self.meta.mode)?);
self.meta.n_layers = self.layers.len();
self.meta.save(&self.root)?;
Ok(())
@@ -100,7 +100,7 @@ impl LayeredMap<()> {
pub fn push_layer(&mut self) -> OLMResult<usize> {
let i = self.layers.len();
let dir = layer_dir(&self.root, i);
Layer::<()>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode)?;
TypedLayer::<()>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode)?;
self.append_layer()?;
Ok(i)
}
@@ -112,7 +112,7 @@ impl LayeredMap<PersistentCompactIntMatrix> {
pub fn push_layer(&mut self, count_of: impl Fn(CanonicalKmer) -> u32) -> OLMResult<usize> {
let i = self.layers.len();
let dir = layer_dir(&self.root, i);
Layer::<PersistentCompactIntMatrix>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode, count_of)?;
TypedLayer::<PersistentCompactIntMatrix>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode, count_of)?;
self.append_layer()?;
Ok(i)
}
+9 -9
View File
@@ -43,7 +43,7 @@ fn canonical_kmer_iter_matches_reader() {
assert_eq!(from_iter, from_reader, "CanonicalKmerIter and UnitigFileReader disagree");
}
// ── Generic `Layer<D>` over dense vs. sparse presence storage ───────────────
// ── Generic `TypedLayer<D>` over dense vs. sparse presence storage ───────────────
#[test]
fn presence_layer_generic_over_sparse_matches_dense() {
@@ -64,11 +64,11 @@ fn presence_layer_generic_over_sparse_matches_dense() {
// kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be
// biologically meaningful, just the same on both sides of the
// dense/sparse comparison below.
Layer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
}).unwrap();
let dense_layer = Layer::<PersistentBitMatrix>::open(dir.path(), &mode).unwrap();
let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path(), &mode).unwrap();
assert!(dense_layer.n_cols() >= 1);
// Build the sparse form directly into the same `presence/` dir the
@@ -81,7 +81,7 @@ fn presence_layer_generic_over_sparse_matches_dense() {
.close()
.unwrap();
let sparse_layer = Layer::<PersistentSparseBitMatrix>::open(dir.path(), &mode).unwrap();
let sparse_layer = TypedLayer::<PersistentSparseBitMatrix>::open(dir.path(), &mode).unwrap();
// Same generic methods, same results, different concrete `D`.
assert_eq!(dense_layer.n_cols(), sparse_layer.n_cols());
@@ -91,7 +91,7 @@ fn presence_layer_generic_over_sparse_matches_dense() {
assert_eq!(dense_layer.sub_matrix(&slots), sparse_layer.sub_matrix(&slots));
// The matrix-agnostic, MPHF-only surface (from the generic
// `impl<D: LayerData> Layer<D>`) must also agree: same kmer set,
// `impl<D: LayerData> TypedLayer<D>`) must also agree: same kmer set,
// looked up through either concrete `D`.
for kmer in all_canonical_kmers(dir.path()) {
assert_eq!(dense_layer.find_slot(kmer), sparse_layer.find_slot(kmer));
@@ -105,9 +105,9 @@ fn count_layer_reports_count_content_and_columnar_storage() {
set_k(4);
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAAACGT"]);
Layer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
TypedLayer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
.unwrap();
let layer = Layer::<PersistentCompactIntMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
assert_eq!(layer.content(), LayerContent::Count);
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
@@ -119,10 +119,10 @@ fn presence_layer_reports_presence_content_and_columnar_storage() {
set_k(4);
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
Layer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
}).unwrap();
let layer = Layer::<PersistentBitMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
assert_eq!(layer.content(), LayerContent::Presence);
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
+4 -4
View File
@@ -125,7 +125,7 @@ fn presence_partition(seqs: &[&[u8]], n_genomes: usize) -> (tempfile::TempDir, L
let mode = IndexMode::Exact;
let layer0 = layer_dir(dir.path(), 0);
write_unitigs_at(&layer0, seqs);
Layer::<PersistentBitMatrix>::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
TypedLayer::<PersistentBitMatrix>::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
}).unwrap();
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
@@ -173,7 +173,7 @@ fn detect_layer_storage_implicit() {
write_unitigs_at(&layer0, &[b"AAAACGT"]);
// Mode-1 build: MPHF + evidence + `layer_meta.json`, no matrix at all —
// must read back as `Presence`/`Implicit`, never a third "empty" state.
Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap();
TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap();
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
let map = LayeredMap::<()>::open(dir.path()).unwrap();
@@ -203,7 +203,7 @@ fn detect_layer_evidence_exact_vs_approx() {
let dir = tempdir().unwrap();
let layer0 = layer_dir(dir.path(), 0);
write_unitigs_at(&layer0, &[b"AAAACGT"]);
Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap();
TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap();
PartitionMeta { n_layers: 1, mode: IndexMode::Exact }.save(dir.path()).unwrap();
let map = LayeredMap::<()>::open(dir.path()).unwrap();
assert_eq!(map.detect_layer_evidence(0).unwrap(), EvidenceKind::Exact);
@@ -212,7 +212,7 @@ fn detect_layer_evidence_exact_vs_approx() {
let dir2 = tempdir().unwrap();
let layer0b = layer_dir(dir2.path(), 0);
write_unitigs_at(&layer0b, &[b"AAAACGT"]);
Layer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap();
TypedLayer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap();
PartitionMeta { n_layers: 1, mode: approx }.save(dir2.path()).unwrap();
let map2 = LayeredMap::<()>::open(dir2.path()).unwrap();
assert_eq!(map2.detect_layer_evidence(0).unwrap(), EvidenceKind::Approx);