Push zunrplorkwkt #70
@@ -133,14 +133,35 @@ needed — `obikindex → obikpartitionner` stays the same direction as before.
|
||||
Full workspace test suite green (0 failed) after, including all 27
|
||||
`obikphylo::siblings` tests.
|
||||
|
||||
Still open: (1)/(2) themselves — `AnyLayer` (or whatever it ends up named;
|
||||
`AnyLayer` was rejected as a placeholder, no replacement chosen yet) and
|
||||
`KmerPartition` (singular, one partition's open layers) are not built yet.
|
||||
`obikphylo::siblings::cache::{Mat, PartitionCache}` and
|
||||
`obikpartitionner::query_layer` (now `obikindex::query_layer`)'s
|
||||
`QueryLayer` still each independently bundle MPHF+matrix — unchanged by
|
||||
this restructuring, which was purely about *where* code lives, not about
|
||||
building the heterogeneous-layer cache itself.
|
||||
## (1) done (2026-08-20): `Layer` is now the heterogeneous handle, `Mat` is gone
|
||||
|
||||
Resolved the naming question left open above. `Layer<D>` (the old
|
||||
generic/monomorphic type) renamed to `TypedLayer<D>` throughout
|
||||
(`obilayeredmap`, `obikindex`, `obikphylo` — 12 files, mechanical) to free
|
||||
`Layer` for the type that's actually meant to be everyone's default
|
||||
handle. `obilayeredmap::content_layer::Layer` (re-exported at the crate
|
||||
root) is that type — `Count(TypedLayer<PersistentCompactIntMatrix>)`/
|
||||
`Presence(TypedLayer<PersistentBitMatrix>)`, `Layer::open` doing the same
|
||||
disk probe `Mat::open` used to, `find_slot`/`index_batch`/`n_cols`/
|
||||
`fill_sub_matrix_carries` dispatching per variant exactly as `Mat` did.
|
||||
|
||||
`obikphylo::siblings::cache::Mat` deleted outright — `PartitionCache` now
|
||||
holds `Vec<Vec<obilayeredmap::Layer>>` directly. The one sibling-specific
|
||||
method `Mat` carried (`iter_minorants_batch`) is not on `obilayeredmap::
|
||||
Layer` (phylo concepts don't belong in `obilayeredmap`) — it's an
|
||||
`impl SiblingLayerExt for obilayeredmap::Layer` in `iter.rs`, dispatching
|
||||
to each variant's existing `impl<D: LayerData> SiblingLayerExt for
|
||||
TypedLayer<D>`.
|
||||
|
||||
Full workspace suite green (0 failed) after, including all 27
|
||||
`obikphylo::siblings` tests.
|
||||
|
||||
Still not built: (2) — `KmerPartition` (singular, one partition's open
|
||||
`Vec<Layer>`) and a multi-partition cache in `obikpartitionner` to replace
|
||||
`obikphylo::siblings::cache::PartitionCache` and `obikindex::query_layer`'s
|
||||
still-separate `QueryLayer` (which still independently bundles MPHF+matrix,
|
||||
2-way not using `Layer` at all). Both remaining consumers now sit one
|
||||
`Layer::open` call away from unifying onto (2) once it exists.
|
||||
|
||||
## The problem
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ use obipipeline::{
|
||||
|
||||
use obidebruinj::GraphDeBruijn;
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::{IndexMode, Layer};
|
||||
use obilayeredmap::{IndexMode, TypedLayer};
|
||||
use obiskio::{SKError, SKResult};
|
||||
|
||||
use crate::common::olm_to_sk;
|
||||
@@ -125,7 +125,7 @@ pub(crate) fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> SKRe
|
||||
let n_kmers = g.len();
|
||||
g.compute_degrees_and_mark_starts();
|
||||
std::fs::create_dir_all(layer_dir)?;
|
||||
let mut uw = Layer::<()>::unitig_writer(layer_dir).map_err(|e| olm_to_sk(e, "graph pipeline"))?;
|
||||
let mut uw = TypedLayer::<()>::unitig_writer(layer_dir).map_err(|e| olm_to_sk(e, "graph pipeline"))?;
|
||||
g.try_for_each_unitig(|unitig| uw.write(unitig))?;
|
||||
uw.close()?;
|
||||
drop(g);
|
||||
@@ -134,7 +134,7 @@ pub(crate) fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> SKRe
|
||||
|
||||
// ── materialize_layer ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Phase 2 (full): write_graph_as_unitigs + `Layer::<()>::build`.
|
||||
/// Phase 2 (full): write_graph_as_unitigs + `TypedLayer::<()>::build`.
|
||||
///
|
||||
/// Returns n_kmers.
|
||||
pub(crate) fn materialize_layer(
|
||||
@@ -145,7 +145,7 @@ pub(crate) fn materialize_layer(
|
||||
) -> SKResult<usize> {
|
||||
let n = write_graph_as_unitigs(g, layer_dir)?;
|
||||
debug!("materialize_layer: unitigs written ({n} kmers), building MPHF");
|
||||
Layer::<()>::build(layer_dir, block_bits, evidence)
|
||||
TypedLayer::<()>::build(layer_dir, block_bits, evidence)
|
||||
.map_err(|e| olm_to_sk(e, "graph pipeline"))?;
|
||||
debug!("materialize_layer: MPHF build done");
|
||||
Ok(n)
|
||||
|
||||
@@ -6,7 +6,7 @@ use epserde::prelude::*;
|
||||
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
|
||||
use obidebruinj::GraphDeBruijn;
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use obilayeredmap::{IndexMode, layer::Layer};
|
||||
use obilayeredmap::{IndexMode, layer::TypedLayer};
|
||||
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
||||
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
||||
|
||||
@@ -96,7 +96,7 @@ impl KmerIndex {
|
||||
let n_kmers =
|
||||
if with_counts {
|
||||
let n = write_graph_as_unitigs(g, &layer_dir)?;
|
||||
Layer::<PersistentCompactIntMatrix>::build(&layer_dir, block_bits, mode, |kmer| {
|
||||
TypedLayer::<PersistentCompactIntMatrix>::build(&layer_dir, block_bits, mode, |kmer| {
|
||||
match (&mphf1_opt, &counts1_opt) {
|
||||
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
|
||||
_ => 1,
|
||||
|
||||
@@ -20,7 +20,7 @@ use tracing::debug;
|
||||
|
||||
use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder};
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::{IndexMode, Layer, LayeredMap, MphfOnly, layer_dir};
|
||||
use obilayeredmap::{IndexMode, TypedLayer, LayeredMap, MphfOnly, layer_dir};
|
||||
use obiskio::{SKError, SKResult, UnitigFileReader};
|
||||
|
||||
use crate::common::{ColBuilder, load_meta, olm_to_sk};
|
||||
@@ -232,7 +232,7 @@ impl KmerIndex {
|
||||
// (all slots true — every kmer in those layers belongs to genome_0).
|
||||
if n_dst_genomes == 1 && mode == MergeMode::Presence {
|
||||
for l in 0..n_dst_layers {
|
||||
Layer::<()>::init_presence_matrix(
|
||||
TypedLayer::<()>::init_presence_matrix(
|
||||
&layer_dir(&dst_index_dir, l),
|
||||
dst_map.layer(l).n(),
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use obilayeredmap::{IndexMode, layer::Layer};
|
||||
use obilayeredmap::{IndexMode, layer::TypedLayer};
|
||||
use obisys::{Reporter, Stage, progress_bar};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
@@ -90,10 +90,10 @@ fn reindex_partition(
|
||||
fn reindex_layer(layer_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
|
||||
match target {
|
||||
IndexMode::Exact => {
|
||||
Layer::<()>::build_exact_evidence(layer_dir, block_bits).map_err(olm_to_oki)?;
|
||||
TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits).map_err(olm_to_oki)?;
|
||||
}
|
||||
IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => {
|
||||
Layer::<()>::build_approx_evidence(layer_dir, *b, *z).map_err(olm_to_oki)?;
|
||||
TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z).map_err(olm_to_oki)?;
|
||||
}
|
||||
}
|
||||
remove_stale_evidence(layer_dir, target)
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
use rayon::prelude::*;
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::meta::IndexMode;
|
||||
use obilayeredmap::{Layer, OLMResult};
|
||||
use obilayeredmap::Layer;
|
||||
use obisys::progress_bar;
|
||||
|
||||
use obikindex::{KmerIndex, OKIResult};
|
||||
|
||||
use super::SiblingAnnex;
|
||||
use super::iter::SiblingLayerExt;
|
||||
|
||||
/// Every partition's already-open layers, built **once** for the whole
|
||||
/// `build_sibling_annex` run and shared (read-only) across every lookup, in
|
||||
/// every source layer, for the rest of the run — not reopened/re-mmap'd per
|
||||
@@ -31,119 +24,14 @@ use super::iter::SiblingLayerExt;
|
||||
/// partition for the entire run, regardless of how many source layers or
|
||||
/// lookups follow.
|
||||
///
|
||||
/// One `Layer<D>` per layer (MPHF + matrix bundled), not a separate
|
||||
/// `MphfLayer` and a separate `PersistentCompactIntMatrix`/
|
||||
/// `PersistentBitMatrix` in parallel arrays — `obilayeredmap::Layer` already
|
||||
/// *is* that bundle, with `find_slot` (MPHF-only, no data read),
|
||||
/// `n_cols`/`sub_matrix`/`fill_sub_matrix` (batched, sorted-internally
|
||||
/// column access) on top of it. Reinventing that pairing here would just be
|
||||
/// going back through the low-level pieces `Layer` already assembles.
|
||||
pub(super) enum Mat {
|
||||
Count(Layer<PersistentCompactIntMatrix>),
|
||||
/// Dense *or* sparse (`pack --sparse`d) presence — `PersistentBitMatrix`
|
||||
/// itself is a 4-way enum (`Columnar`/`Packed`/`Sparse`/`Implicit`) that
|
||||
/// already auto-detects sparse storage in its own `open()` (checks
|
||||
/// `presence/sparse_meta.json`) and dispatches every method
|
||||
/// (`row`/`fill_row`/`nonzero_iter`/...) across all four internally —
|
||||
/// see `DevDocMD/implementation/partition_layer_cache.md`. A separate
|
||||
/// `SparsePresence(Layer<PersistentSparseBitMatrix>)` arm used to exist
|
||||
/// here, opened via its own `presence/is_multi.prsb` check; it
|
||||
/// predated `PersistentBitMatrix` growing native sparse support and
|
||||
/// was pure duplication by the time it was removed — every method
|
||||
/// below dispatched it identically to this arm.
|
||||
Presence(Layer<PersistentBitMatrix>),
|
||||
}
|
||||
|
||||
impl Mat {
|
||||
/// Open one layer's matrix, auto-detecting count vs. presence from what
|
||||
/// is actually on disk — the single source of truth for *every* caller
|
||||
/// that opens a layer's own matrix (this module's cross-partition
|
||||
/// [`PartitionCache::build`] and `family_scan::scan_layer_families`'s
|
||||
/// own-layer lookup alike), so the two can never disagree. Sparse vs.
|
||||
/// dense presence storage is `PersistentBitMatrix::open`'s own concern
|
||||
/// (see the [`Presence`](Mat::Presence) variant's docs), not decided
|
||||
/// here.
|
||||
pub(super) fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult<Self> {
|
||||
if with_counts && layer_dir.join("counts").exists() {
|
||||
return Layer::<PersistentCompactIntMatrix>::open(layer_dir, mode).map(Mat::Count);
|
||||
}
|
||||
Layer::<PersistentBitMatrix>::open(layer_dir, mode).map(Mat::Presence)
|
||||
}
|
||||
|
||||
fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
|
||||
match self {
|
||||
Mat::Count(l) => l.find_slot(kmer),
|
||||
Mat::Presence(l) => l.find_slot(kmer),
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw MPHF batch lookup: kmer → slot, no membership check — for
|
||||
/// callers that already know every kmer is a member of *this* layer
|
||||
/// (e.g. it came from this layer's own `iter_minorants_batch`), so the
|
||||
/// evidence check `find_slot`/`find` would perform is redundant work.
|
||||
/// See `DevDocMD/architecture/siblings.md`: iteration-pipeline kmers use
|
||||
/// `index`, never `find`.
|
||||
pub(super) fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
|
||||
match self {
|
||||
Mat::Count(l) => l.index_batch(kmers),
|
||||
Mat::Presence(l) => l.index_batch(kmers),
|
||||
}
|
||||
}
|
||||
|
||||
/// This layer's own `SiblingLayerExt::iter_minorants_batch` — dispatch
|
||||
/// only, both arms return the same concrete `MinorantBatchIter` (it
|
||||
/// doesn't depend on `D`), so no boxing is needed.
|
||||
pub(super) fn iter_minorants_batch(
|
||||
&self,
|
||||
annex: std::sync::Arc<SiblingAnnex>,
|
||||
batch_size: usize,
|
||||
) -> super::iter::MinorantBatchIter {
|
||||
match self {
|
||||
Mat::Count(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
Mat::Presence(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn n_cols(&self) -> usize {
|
||||
match self {
|
||||
Mat::Count(l) => l.n_cols(),
|
||||
Mat::Presence(l) => l.n_cols(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` =
|
||||
/// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must
|
||||
/// have one entry per genome column, each resized to `slots.len()`.
|
||||
///
|
||||
/// Delegates entirely to `Layer<D>::fill_sub_matrix`, which sorts
|
||||
/// `slots` once internally for a sequential mmap sweep per column, then
|
||||
/// restores the original order — the same discipline this call site
|
||||
/// (and `find_presence_batch`) used to hand-roll with its own sort +
|
||||
/// genome-major loop. `Count` still needs one intermediate
|
||||
/// `Vec<Vec<u32>>` fetch (the underlying store only has an int
|
||||
/// sub-matrix, not a bool one), converted to presence (`!= 0`) in place
|
||||
/// — the sort/sequential-access win is unaffected, just one extra
|
||||
/// allocation pass over already-in-hand data.
|
||||
pub(super) fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec<bool>]) {
|
||||
match self {
|
||||
Mat::Presence(l) => l.fill_sub_matrix(slots, out),
|
||||
Mat::Count(l) => {
|
||||
let mut counts: Vec<Vec<u32>> = out.iter().map(|_| Vec::new()).collect();
|
||||
l.fill_sub_matrix(slots, &mut counts);
|
||||
for (o, c) in out.iter_mut().zip(counts.iter()) {
|
||||
o.clear();
|
||||
o.extend(c.iter().map(|&v| v != 0));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `mats[partition][layer]` holds `obilayeredmap::Layer` — the
|
||||
/// format-erased `Count`/`Presence` handle, not a sibling-specific type:
|
||||
/// this module used to bundle its own `Mat` enum here, duplicating exactly
|
||||
/// what `Layer` now does one crate down (see
|
||||
/// `DevDocMD/implementation/partition_layer_cache.md`). Its
|
||||
/// sibling-specific extension (`iter_minorants_batch`) lives in `iter.rs`.
|
||||
pub(super) struct PartitionCache {
|
||||
/// `mats[partition][layer]` = that partition's opened layers. Used by
|
||||
/// both [`obikindex::KmerIndex::build_sibling_annex`] and
|
||||
/// [`obikindex::KmerIndex::sibling_annex_stats`].
|
||||
mats: Vec<Vec<Mat>>,
|
||||
mats: Vec<Vec<Layer>>,
|
||||
/// Whether every `FamilyMask` field in this index's sibling annexes can
|
||||
/// be trusted as a real layer index (`n_layers <= 7`, the same decision
|
||||
/// `build_layer_sibling_annex` makes when writing them) — computed once
|
||||
@@ -163,9 +51,9 @@ impl PartitionCache {
|
||||
with_counts: bool,
|
||||
) -> OKIResult<Self> {
|
||||
let pb = progress_bar("open_partitions", n_parts as u64, "partitions");
|
||||
let built: Vec<(Vec<Mat>, usize)> = (0..n_parts)
|
||||
let built: Vec<(Vec<Layer>, usize)> = (0..n_parts)
|
||||
.into_par_iter()
|
||||
.map(|part| -> OKIResult<(Vec<Mat>, usize)> {
|
||||
.map(|part| -> OKIResult<(Vec<Layer>, usize)> {
|
||||
let index_dir = index.index_dir(part);
|
||||
if !index_dir.exists() {
|
||||
pb.inc(1);
|
||||
@@ -174,7 +62,7 @@ impl PartitionCache {
|
||||
let meta = index.partition_meta(part)?;
|
||||
let mut mats = Vec::with_capacity(meta.n_layers);
|
||||
for l in 0..meta.n_layers {
|
||||
let Ok(mat) = Mat::open(&index.layer_dir(part, l), &meta.mode, with_counts)
|
||||
let Ok(mat) = Layer::open(&index.layer_dir(part, l), &meta.mode, with_counts)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
@@ -331,7 +219,7 @@ impl PartitionCache {
|
||||
/// `hits` gets its slots (evidence-probed vs. trusted `index_batch`) — this
|
||||
/// is everything after that.
|
||||
fn resolve_layer_hits(
|
||||
mat: &Mat,
|
||||
mat: &Layer,
|
||||
hits: &[(usize, usize, u8)],
|
||||
n_genomes: usize,
|
||||
on_hit: &mut impl FnMut(usize, u8, usize),
|
||||
|
||||
@@ -60,9 +60,11 @@ use obipipeline::{ThrottleGuard, throttle};
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
use obikindex::KmerIndex;
|
||||
|
||||
use super::cache::{Mat, PartitionCache};
|
||||
use obilayeredmap::Layer;
|
||||
|
||||
use super::cache::PartitionCache;
|
||||
use super::helpers::central_base;
|
||||
use super::iter::SiblingEntry;
|
||||
use super::iter::{SiblingEntry, SiblingLayerExt};
|
||||
use super::{olm_to_ok, FamilyMask, SiblingAnnex, ANNEX_FILE_NAME};
|
||||
|
||||
/// Families per batch — see the module docs for the memory-vs-per-partition-
|
||||
@@ -129,10 +131,10 @@ pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult<Vec<PathBuf>> {
|
||||
/// generating this layer's batches — opened once, not per batch. No `cache`
|
||||
/// here: generation never touches the cross-partition cache, only this
|
||||
/// layer's own already-open matrix. No separate MPHF/`slot_kmer` either —
|
||||
/// `mat` (a `Layer<D>`) already bundles the MPHF, and each `SiblingEntry`
|
||||
/// `mat` (a `TypedLayer<D>`) already bundles the MPHF, and each `SiblingEntry`
|
||||
/// arrives with its kmer and mask already in hand from `iter_minorants_batch`.
|
||||
struct LayerCtx {
|
||||
mat: Mat,
|
||||
mat: Layer,
|
||||
n_parts: usize,
|
||||
n_genomes: usize,
|
||||
n_cols: usize,
|
||||
@@ -193,7 +195,7 @@ pub(super) fn scan_layer_families(
|
||||
let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?;
|
||||
let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?);
|
||||
|
||||
let mat = Mat::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?;
|
||||
let mat = Layer::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?;
|
||||
let n_cols = mat.n_cols().min(n_genomes);
|
||||
|
||||
let ctx = Arc::new(LayerCtx { mat, n_parts, n_genomes, n_cols, k });
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! Phylo/sibling-domain iteration over a layer — an extension trait, not a
|
||||
//! new field on `MphfLayer`/`Layer<D>`: "family"/"minorant" are phylo
|
||||
//! new field on `MphfLayer`/`TypedLayer<D>`: "family"/"minorant" are phylo
|
||||
//! concepts, `obilayeredmap` stays kmer/slot-mapping only (see
|
||||
//! `DevDocMD/architecture/siblings.md`).
|
||||
//!
|
||||
@@ -29,7 +29,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::{KmerIter, Layer, LayerData};
|
||||
use obilayeredmap::{KmerIter, TypedLayer, LayerData};
|
||||
|
||||
use super::{FamilyMask, SiblingAnnex};
|
||||
|
||||
@@ -128,10 +128,10 @@ fn collect_batch<I: Iterator>(inner: &mut I, batch_size: usize) -> Option<Vec<I:
|
||||
if batch.is_empty() { None } else { Some(batch) }
|
||||
}
|
||||
|
||||
/// Adds phylo/sibling iteration to any `Layer<D>` — the extension that
|
||||
/// Adds phylo/sibling iteration to any `TypedLayer<D>` — the extension that
|
||||
/// turns a plain layer into a "sibling layer". Generic over `D`
|
||||
/// (`LayerData`) rather than implemented once per matrix kind: kmer
|
||||
/// iteration doesn't depend on the data payload, and `Layer<D>` already
|
||||
/// iteration doesn't depend on the data payload, and `TypedLayer<D>` already
|
||||
/// delegates `iter_kmers`/`index`/`index_batch` to its inner MPHF for every
|
||||
/// `D` — reusing that instead of going back through a separate `MphfLayer`.
|
||||
pub trait SiblingLayerExt {
|
||||
@@ -154,7 +154,7 @@ pub trait SiblingLayerExt {
|
||||
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter;
|
||||
}
|
||||
|
||||
impl<D: LayerData> SiblingLayerExt for Layer<D> {
|
||||
impl<D: LayerData> SiblingLayerExt for TypedLayer<D> {
|
||||
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
|
||||
SiblingIter { kmers: self.iter_kmers(), annex, order: 0 }
|
||||
}
|
||||
@@ -171,3 +171,39 @@ impl<D: LayerData> SiblingLayerExt for Layer<D> {
|
||||
MinorantBatchIter { inner: self.iter_minorants(annex), batch_size }
|
||||
}
|
||||
}
|
||||
|
||||
/// Same extension, over `obilayeredmap::Layer` (the format-erased
|
||||
/// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch
|
||||
/// only, both arms return the same concrete iterator types (they don't
|
||||
/// depend on which `D` is inside), so no boxing is needed. `obilayeredmap`
|
||||
/// itself can't implement this: "family"/"minorant" are phylo concepts, it
|
||||
/// stays kmer/slot-mapping only (see the module docs above).
|
||||
impl SiblingLayerExt for obilayeredmap::Layer {
|
||||
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_siblings(annex),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_siblings(annex),
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_siblings_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> SiblingBatchIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size),
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_minorants(&self, annex: Arc<SiblingAnnex>) -> MinorantIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_minorants(annex),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_minorants(annex),
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,9 +156,9 @@ fn sibling_annex_works_after_pack_sparse() {
|
||||
// `pack --sparse` (`KmerIndex::pack_matrices(true)`) run on the merged
|
||||
// index before building the sibling annex — proves `PartitionCache`'s
|
||||
// sparse-detection (`Mat::SparsePresence`, gated on `presence/
|
||||
// is_multi.prsb`) and the generic `Layer<D>` methods it relies on
|
||||
// is_multi.prsb`) and the generic `TypedLayer<D>` methods it relies on
|
||||
// actually round-trip through the real build pipeline, not just the
|
||||
// unit-level `Layer<PersistentSparseBitMatrix>` tests in
|
||||
// unit-level `TypedLayer<PersistentSparseBitMatrix>` tests in
|
||||
// `obilayeredmap`.
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
//! [`Layer`] — the format-erased layer handle every caller outside this
|
||||
//! crate should reach for. `TypedLayer<D>` (`layer.rs`) is monomorphic: a
|
||||
//! `Vec<TypedLayer<D>>` needs one `D` fixed at compile time for every
|
||||
//! element, which real partitions don't respect (`with_counts` can differ
|
||||
//! layer to layer across merges; see
|
||||
//! `DevDocMD/implementation/partition_layer_cache.md`). `Layer` is the
|
||||
//! sum type that lets a partition's layers be held together regardless —
|
||||
//! `Count`/`Presence` each wrap the one concrete `TypedLayer<D>` that
|
||||
//! content implies, chosen by [`Layer::open`]'s own disk probe.
|
||||
//!
|
||||
//! This is exactly the shape `obikphylo::siblings::cache::Mat` used to
|
||||
//! reimplement locally, minus its one sibling-specific method
|
||||
//! (`iter_minorants_batch`, which stays an extension trait over there —
|
||||
//! `obilayeredmap` has no business knowing about sibling annexes).
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
||||
use obikseq::CanonicalKmer;
|
||||
|
||||
use crate::error::OLMResult;
|
||||
use crate::layer::{LayerContent, TypedLayer, COUNTS_DIR};
|
||||
use crate::meta::IndexMode;
|
||||
use crate::mphf_layer::EvidenceKind;
|
||||
|
||||
/// One layer, its content (`Count`/`Presence`) resolved at open time rather
|
||||
/// than at compile time — see the module docs.
|
||||
pub enum Layer {
|
||||
Count(TypedLayer<PersistentCompactIntMatrix>),
|
||||
Presence(TypedLayer<PersistentBitMatrix>),
|
||||
}
|
||||
|
||||
impl Layer {
|
||||
/// Open one layer, auto-detecting count vs. presence from what is
|
||||
/// actually on disk (`counts/` present and wanted, else presence) — the
|
||||
/// single source of truth every caller that opens a layer's own matrix
|
||||
/// should share, so two callers can never disagree about which format a
|
||||
/// given layer is. Dense vs. sparse presence storage is
|
||||
/// `PersistentBitMatrix::open`'s own concern (it's a 4-way
|
||||
/// `Columnar`/`Packed`/`Sparse`/`Implicit` enum internally), not decided
|
||||
/// here.
|
||||
pub fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult<Self> {
|
||||
if with_counts && layer_dir.join(COUNTS_DIR).exists() {
|
||||
return TypedLayer::<PersistentCompactIntMatrix>::open(layer_dir, mode).map(Layer::Count);
|
||||
}
|
||||
TypedLayer::<PersistentBitMatrix>::open(layer_dir, mode).map(Layer::Presence)
|
||||
}
|
||||
|
||||
pub fn content(&self) -> LayerContent {
|
||||
match self {
|
||||
Layer::Count(_) => LayerContent::Count,
|
||||
Layer::Presence(_) => LayerContent::Presence,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn evidence_kind(&self) -> EvidenceKind {
|
||||
match self {
|
||||
Layer::Count(l) => l.evidence_kind(),
|
||||
Layer::Presence(l) => l.evidence_kind(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn n(&self) -> usize {
|
||||
match self {
|
||||
Layer::Count(l) => l.n(),
|
||||
Layer::Presence(l) => l.n(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
|
||||
match self {
|
||||
Layer::Count(l) => l.find_slot(kmer),
|
||||
Layer::Presence(l) => l.find_slot(kmer),
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw MPHF batch lookup: kmer → slot, no membership check — for
|
||||
/// callers that already know every kmer is a member of *this* layer, so
|
||||
/// the evidence check `find_slot`/`find` would perform is redundant.
|
||||
pub fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
|
||||
match self {
|
||||
Layer::Count(l) => l.index_batch(kmers),
|
||||
Layer::Presence(l) => l.index_batch(kmers),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn n_cols(&self) -> usize {
|
||||
match self {
|
||||
Layer::Count(l) => l.n_cols(),
|
||||
Layer::Presence(l) => l.n_cols(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` =
|
||||
/// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must
|
||||
/// have one entry per genome column, each resized to `slots.len()`.
|
||||
/// `Count` needs one intermediate `Vec<Vec<u32>>` fetch (the underlying
|
||||
/// store only has an int sub-matrix, not a bool one), converted to
|
||||
/// presence (`!= 0`) in place.
|
||||
pub fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec<bool>]) {
|
||||
match self {
|
||||
Layer::Presence(l) => l.fill_sub_matrix(slots, out),
|
||||
Layer::Count(l) => {
|
||||
let mut counts: Vec<Vec<u32>> = out.iter().map(|_| Vec::new()).collect();
|
||||
l.fill_sub_matrix(slots, &mut counts);
|
||||
for (o, c) in out.iter_mut().zip(counts.iter()) {
|
||||
o.clear();
|
||||
o.extend(c.iter().map(|&v| v != 0));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,7 @@ pub trait LayerData: Sized {
|
||||
fn read(&self, slot: usize) -> Self::Item;
|
||||
}
|
||||
|
||||
/// Layer directory path for layer `i` within a partition's index root — the
|
||||
/// TypedLayer directory path for layer `i` within a partition's index root — the
|
||||
/// single source of truth for the on-disk `layer_N` naming convention
|
||||
/// (mirrors what `LayeredMap::open`/`push_layer` already use internally).
|
||||
///
|
||||
@@ -43,7 +43,7 @@ pub fn layer_dir(root: &Path, i: usize) -> PathBuf {
|
||||
/// Opens layer `i`'s data only, skipping the MPHF — for callers that only
|
||||
/// need matrix-level operations (distance traits, column weights, group
|
||||
/// filters, sub-matrix extraction) and never look up a kmer for this layer.
|
||||
/// `Layer<D>::open` always pays for the MPHF too (and doesn't expose `data`
|
||||
/// `TypedLayer<D>::open` always pays for the MPHF too (and doesn't expose `data`
|
||||
/// once open), so it's the wrong tool for these; this is the other half of
|
||||
/// the same `D::open(layer_dir)` call, without the MPHF alongside it.
|
||||
///
|
||||
@@ -89,7 +89,7 @@ impl LayerData for PersistentSparseBitMatrix {
|
||||
|
||||
/// What a layer's data matrix represents — orthogonal to *how* it's stored
|
||||
/// on disk (`obicompactvec::StorageKind`'s concern). Only meaningful for
|
||||
/// `D` that actually carry a matrix: `Layer<()>` (mode 1, set membership,
|
||||
/// `D` that actually carry a matrix: `TypedLayer<()>` (mode 1, set membership,
|
||||
/// no matrix at all) has neither a `LayerContent` nor a `content()` method
|
||||
/// — it's a write-time-only state, never a queryable content. Once a layer
|
||||
/// is closed, "no matrix file" reads back as `Presence` via
|
||||
@@ -115,7 +115,7 @@ impl LayerContent {
|
||||
}
|
||||
|
||||
/// Implemented only by `D` that carry a real matrix — gates
|
||||
/// [`Layer::content`](Layer::content) so `Layer<()>` doesn't get it.
|
||||
/// [`TypedLayer::content`](TypedLayer::content) so `TypedLayer<()>` doesn't get it.
|
||||
pub trait HasLayerContent {
|
||||
const CONTENT: LayerContent;
|
||||
}
|
||||
@@ -135,7 +135,7 @@ impl HasLayerContent for PersistentSparseBitMatrix {
|
||||
// ── StorageKind passthrough ────────────────────────────────────────────────────
|
||||
|
||||
/// Implemented only by `D` that expose their own `storage_kind()` — gates
|
||||
/// [`Layer::storage_kind`]. `PersistentSparseBitMatrix` has a single,
|
||||
/// [`TypedLayer::storage_kind`]. `PersistentSparseBitMatrix` has a single,
|
||||
/// fixed on-disk format (no `Columnar`/`Packed`/`Implicit` variants of its
|
||||
/// own), so it's deliberately not given an impl: there's nothing to report
|
||||
/// beyond what `content()` already says.
|
||||
@@ -157,7 +157,7 @@ impl HasStorageKind for PersistentBitMatrix {
|
||||
|
||||
// ── Structures ────────────────────────────────────────────────────────────────
|
||||
|
||||
pub struct Layer<D: LayerData = ()> {
|
||||
pub struct TypedLayer<D: LayerData = ()> {
|
||||
mphf: MphfLayer,
|
||||
data: D,
|
||||
}
|
||||
@@ -169,7 +169,7 @@ pub struct Hit<T = ()> {
|
||||
|
||||
// ── Common read path ──────────────────────────────────────────────────────────
|
||||
|
||||
impl<D: LayerData> Layer<D> {
|
||||
impl<D: LayerData> TypedLayer<D> {
|
||||
pub fn open(path: &Path, mode: &IndexMode) -> OLMResult<Self> {
|
||||
let mphf = MphfLayer::open(path, mode)?;
|
||||
let data = D::open(path)?;
|
||||
@@ -249,7 +249,7 @@ impl<D: LayerData> Layer<D> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<D: LayerData + HasLayerContent> Layer<D> {
|
||||
impl<D: LayerData + HasLayerContent> TypedLayer<D> {
|
||||
/// This already-open layer's content (`Count`/`Presence`) — a
|
||||
/// compile-time fact about `D`, not a runtime read.
|
||||
pub const fn content(&self) -> LayerContent {
|
||||
@@ -257,7 +257,7 @@ impl<D: LayerData + HasLayerContent> Layer<D> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<D: LayerData + HasStorageKind> Layer<D> {
|
||||
impl<D: LayerData + HasStorageKind> TypedLayer<D> {
|
||||
/// This already-open layer's storage format — delegates to `D`'s own
|
||||
/// `storage_kind()`, reading a discriminant already in memory.
|
||||
pub fn storage_kind(&self) -> obicompactvec::StorageKind {
|
||||
@@ -267,7 +267,7 @@ impl<D: LayerData + HasStorageKind> Layer<D> {
|
||||
|
||||
// ── Mode 1 — set membership ───────────────────────────────────────────────────
|
||||
|
||||
impl Layer<()> {
|
||||
impl TypedLayer<()> {
|
||||
pub fn build(out_dir: &Path, block_bits: u8, mode: &IndexMode) -> OLMResult<usize> {
|
||||
MphfLayer::build(out_dir, block_bits, mode, &mut |_, _| Ok(()))
|
||||
}
|
||||
@@ -284,7 +284,7 @@ impl Layer<()> {
|
||||
|
||||
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
|
||||
|
||||
impl Layer<PersistentCompactIntMatrix> {
|
||||
impl TypedLayer<PersistentCompactIntMatrix> {
|
||||
pub fn build(
|
||||
out_dir: &Path,
|
||||
block_bits: u8,
|
||||
@@ -317,7 +317,7 @@ impl Layer<PersistentCompactIntMatrix> {
|
||||
|
||||
// ── Mode 2 — count matrix column append ──────────────────────────────────────
|
||||
|
||||
impl Layer<PersistentCompactIntMatrix> {
|
||||
impl TypedLayer<PersistentCompactIntMatrix> {
|
||||
pub fn append_genome_column(
|
||||
layer_dir: &Path,
|
||||
value_of: impl Fn(usize) -> u32,
|
||||
@@ -357,14 +357,14 @@ impl Layer<PersistentCompactIntMatrix> {
|
||||
//
|
||||
// `n_cols`/`sub_matrix`/`fill_sub_matrix` are identical in shape for any
|
||||
// `D` satisfying `BinaryMatrix` — genuinely generic (not just two
|
||||
// near-identical impl blocks) so `Layer<PersistentSparseBitMatrix>` gets
|
||||
// them for free, matching `Layer<PersistentBitMatrix>` at the same call
|
||||
// near-identical impl blocks) so `TypedLayer<PersistentSparseBitMatrix>` gets
|
||||
// them for free, matching `TypedLayer<PersistentBitMatrix>` at the same call
|
||||
// sites (see `obikphylo::siblings::cache::Mat`). Construction
|
||||
// (`append_genome_column`/`build_presence`) stays `PersistentBitMatrix`-only
|
||||
// below: sparse matrices aren't built column-by-column, they're built
|
||||
// row-by-row from an already-built dense layer
|
||||
// (`PersistentSparseBitMatrixBuilder::build_from_dense`).
|
||||
impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix> Layer<D> {
|
||||
impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix> TypedLayer<D> {
|
||||
/// Number of genome columns in this layer's presence matrix — see
|
||||
/// `PersistentBitMatrix::n_cols`'s docs for the `Implicit` mono-genome
|
||||
/// special case (always reports `1`, regardless of the index's real
|
||||
@@ -395,7 +395,7 @@ impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix> Layer<D> {
|
||||
}
|
||||
}
|
||||
|
||||
impl Layer<PersistentBitMatrix> {
|
||||
impl TypedLayer<PersistentBitMatrix> {
|
||||
pub fn append_genome_column(
|
||||
layer_dir: &Path,
|
||||
value_of: impl Fn(usize) -> bool,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod content_layer;
|
||||
pub mod error;
|
||||
pub mod evidence;
|
||||
pub mod fingerprint;
|
||||
@@ -7,8 +8,9 @@ pub mod map;
|
||||
pub mod meta;
|
||||
pub(crate) mod mphf_layer;
|
||||
|
||||
pub use content_layer::Layer;
|
||||
pub use error::{OLMError, OLMResult};
|
||||
pub use layer::{layer_dir, open_data, HasLayerContent, HasStorageKind, Hit, Layer, LayerContent, LayerData};
|
||||
pub use layer::{layer_dir, open_data, HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer};
|
||||
pub use layered_store::LayeredStore;
|
||||
pub use map::LayeredMap;
|
||||
pub use meta::{IndexMode, PartitionMeta};
|
||||
|
||||
@@ -7,7 +7,7 @@ use obikseq::CanonicalKmer;
|
||||
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
|
||||
|
||||
use crate::error::{OLMError, OLMResult};
|
||||
use crate::layer::{layer_dir, Hit, Layer, LayerContent, LayerData};
|
||||
use crate::layer::{layer_dir, Hit, TypedLayer, LayerContent, LayerData};
|
||||
use crate::meta::{IndexMode, PartitionMeta};
|
||||
use crate::mphf_layer::EvidenceKind;
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::mphf_layer::EvidenceKind;
|
||||
pub struct LayeredMap<D: LayerData = ()> {
|
||||
root: PathBuf,
|
||||
meta: PartitionMeta,
|
||||
layers: Vec<Layer<D>>,
|
||||
layers: Vec<TypedLayer<D>>,
|
||||
}
|
||||
|
||||
// ── Common methods ────────────────────────────────────────────────────────────
|
||||
@@ -30,7 +30,7 @@ impl<D: LayerData> LayeredMap<D> {
|
||||
pub fn open(root: &Path) -> OLMResult<Self> {
|
||||
let meta = PartitionMeta::load(root)?;
|
||||
let layers = (0..meta.n_layers)
|
||||
.map(|i| Layer::<D>::open(&layer_dir(root, i), &meta.mode))
|
||||
.map(|i| TypedLayer::<D>::open(&layer_dir(root, i), &meta.mode))
|
||||
.collect::<OLMResult<Vec<_>>>()?;
|
||||
Ok(Self { root: root.to_owned(), meta, layers })
|
||||
}
|
||||
@@ -44,18 +44,18 @@ impl<D: LayerData> LayeredMap<D> {
|
||||
}
|
||||
|
||||
pub fn n_layers(&self) -> usize { self.layers.len() }
|
||||
pub fn layer(&self, i: usize) -> &Layer<D> { &self.layers[i] }
|
||||
pub fn layer(&self, i: usize) -> &TypedLayer<D> { &self.layers[i] }
|
||||
pub fn mode(&self) -> &IndexMode { &self.meta.mode }
|
||||
|
||||
/// Layer `i`'s content (`Count`/`Presence`) — a lightweight disk probe,
|
||||
/// TypedLayer `i`'s content (`Count`/`Presence`) — a lightweight disk probe,
|
||||
/// no MPHF/matrix opened. For picking a `D` before committing to
|
||||
/// [`Layer::open`], or for reporting/diagnostics over an already-open
|
||||
/// [`TypedLayer::open`], or for reporting/diagnostics over an already-open
|
||||
/// `LayeredMap` without re-opening every layer under a different `D`.
|
||||
pub fn detect_layer_content(&self, i: usize) -> LayerContent {
|
||||
LayerContent::detect(&layer_dir(&self.root, i))
|
||||
}
|
||||
|
||||
/// Layer `i`'s storage format — a lightweight disk probe, no
|
||||
/// TypedLayer `i`'s storage format — a lightweight disk probe, no
|
||||
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
|
||||
pub fn detect_layer_storage(&self, i: usize) -> OLMResult<StorageKind> {
|
||||
let dir = layer_dir(&self.root, i);
|
||||
@@ -65,7 +65,7 @@ impl<D: LayerData> LayeredMap<D> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Layer `i`'s evidence mode — a lightweight disk probe, no
|
||||
/// TypedLayer `i`'s evidence mode — a lightweight disk probe, no
|
||||
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
|
||||
pub fn detect_layer_evidence(&self, i: usize) -> OLMResult<EvidenceKind> {
|
||||
EvidenceKind::detect(&layer_dir(&self.root, i))
|
||||
@@ -81,13 +81,13 @@ impl<D: LayerData> LayeredMap<D> {
|
||||
|
||||
pub fn next_layer_writer(&self) -> OLMResult<UnitigFileWriter> {
|
||||
let dir = layer_dir(&self.root, self.layers.len());
|
||||
Layer::<D>::unitig_writer(&dir)
|
||||
TypedLayer::<D>::unitig_writer(&dir)
|
||||
}
|
||||
|
||||
fn append_layer(&mut self) -> OLMResult<()> {
|
||||
let i = self.layers.len();
|
||||
let dir = layer_dir(&self.root, i);
|
||||
self.layers.push(Layer::<D>::open(&dir, &self.meta.mode)?);
|
||||
self.layers.push(TypedLayer::<D>::open(&dir, &self.meta.mode)?);
|
||||
self.meta.n_layers = self.layers.len();
|
||||
self.meta.save(&self.root)?;
|
||||
Ok(())
|
||||
@@ -100,7 +100,7 @@ impl LayeredMap<()> {
|
||||
pub fn push_layer(&mut self) -> OLMResult<usize> {
|
||||
let i = self.layers.len();
|
||||
let dir = layer_dir(&self.root, i);
|
||||
Layer::<()>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode)?;
|
||||
TypedLayer::<()>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode)?;
|
||||
self.append_layer()?;
|
||||
Ok(i)
|
||||
}
|
||||
@@ -112,7 +112,7 @@ impl LayeredMap<PersistentCompactIntMatrix> {
|
||||
pub fn push_layer(&mut self, count_of: impl Fn(CanonicalKmer) -> u32) -> OLMResult<usize> {
|
||||
let i = self.layers.len();
|
||||
let dir = layer_dir(&self.root, i);
|
||||
Layer::<PersistentCompactIntMatrix>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode, count_of)?;
|
||||
TypedLayer::<PersistentCompactIntMatrix>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode, count_of)?;
|
||||
self.append_layer()?;
|
||||
Ok(i)
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ fn canonical_kmer_iter_matches_reader() {
|
||||
assert_eq!(from_iter, from_reader, "CanonicalKmerIter and UnitigFileReader disagree");
|
||||
}
|
||||
|
||||
// ── Generic `Layer<D>` over dense vs. sparse presence storage ───────────────
|
||||
// ── Generic `TypedLayer<D>` over dense vs. sparse presence storage ───────────────
|
||||
|
||||
#[test]
|
||||
fn presence_layer_generic_over_sparse_matches_dense() {
|
||||
@@ -64,11 +64,11 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
||||
// kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be
|
||||
// biologically meaningful, just the same on both sides of the
|
||||
// dense/sparse comparison below.
|
||||
Layer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
|
||||
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
|
||||
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
||||
}).unwrap();
|
||||
|
||||
let dense_layer = Layer::<PersistentBitMatrix>::open(dir.path(), &mode).unwrap();
|
||||
let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path(), &mode).unwrap();
|
||||
assert!(dense_layer.n_cols() >= 1);
|
||||
|
||||
// Build the sparse form directly into the same `presence/` dir the
|
||||
@@ -81,7 +81,7 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
||||
.close()
|
||||
.unwrap();
|
||||
|
||||
let sparse_layer = Layer::<PersistentSparseBitMatrix>::open(dir.path(), &mode).unwrap();
|
||||
let sparse_layer = TypedLayer::<PersistentSparseBitMatrix>::open(dir.path(), &mode).unwrap();
|
||||
|
||||
// Same generic methods, same results, different concrete `D`.
|
||||
assert_eq!(dense_layer.n_cols(), sparse_layer.n_cols());
|
||||
@@ -91,7 +91,7 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
||||
assert_eq!(dense_layer.sub_matrix(&slots), sparse_layer.sub_matrix(&slots));
|
||||
|
||||
// The matrix-agnostic, MPHF-only surface (from the generic
|
||||
// `impl<D: LayerData> Layer<D>`) must also agree: same kmer set,
|
||||
// `impl<D: LayerData> TypedLayer<D>`) must also agree: same kmer set,
|
||||
// looked up through either concrete `D`.
|
||||
for kmer in all_canonical_kmers(dir.path()) {
|
||||
assert_eq!(dense_layer.find_slot(kmer), sparse_layer.find_slot(kmer));
|
||||
@@ -105,9 +105,9 @@ fn count_layer_reports_count_content_and_columnar_storage() {
|
||||
set_k(4);
|
||||
let dir = tempdir().unwrap();
|
||||
write_unitigs(dir.path(), &[b"AAAACGT"]);
|
||||
Layer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
|
||||
TypedLayer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
|
||||
.unwrap();
|
||||
let layer = Layer::<PersistentCompactIntMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
|
||||
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
|
||||
|
||||
assert_eq!(layer.content(), LayerContent::Count);
|
||||
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
||||
@@ -119,10 +119,10 @@ fn presence_layer_reports_presence_content_and_columnar_storage() {
|
||||
set_k(4);
|
||||
let dir = tempdir().unwrap();
|
||||
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
|
||||
Layer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
|
||||
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
|
||||
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
||||
}).unwrap();
|
||||
let layer = Layer::<PersistentBitMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
|
||||
let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
|
||||
|
||||
assert_eq!(layer.content(), LayerContent::Presence);
|
||||
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
||||
|
||||
@@ -125,7 +125,7 @@ fn presence_partition(seqs: &[&[u8]], n_genomes: usize) -> (tempfile::TempDir, L
|
||||
let mode = IndexMode::Exact;
|
||||
let layer0 = layer_dir(dir.path(), 0);
|
||||
write_unitigs_at(&layer0, seqs);
|
||||
Layer::<PersistentBitMatrix>::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
|
||||
TypedLayer::<PersistentBitMatrix>::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
|
||||
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
||||
}).unwrap();
|
||||
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
|
||||
@@ -173,7 +173,7 @@ fn detect_layer_storage_implicit() {
|
||||
write_unitigs_at(&layer0, &[b"AAAACGT"]);
|
||||
// Mode-1 build: MPHF + evidence + `layer_meta.json`, no matrix at all —
|
||||
// must read back as `Presence`/`Implicit`, never a third "empty" state.
|
||||
Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap();
|
||||
TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap();
|
||||
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
|
||||
let map = LayeredMap::<()>::open(dir.path()).unwrap();
|
||||
|
||||
@@ -203,7 +203,7 @@ fn detect_layer_evidence_exact_vs_approx() {
|
||||
let dir = tempdir().unwrap();
|
||||
let layer0 = layer_dir(dir.path(), 0);
|
||||
write_unitigs_at(&layer0, &[b"AAAACGT"]);
|
||||
Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap();
|
||||
TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap();
|
||||
PartitionMeta { n_layers: 1, mode: IndexMode::Exact }.save(dir.path()).unwrap();
|
||||
let map = LayeredMap::<()>::open(dir.path()).unwrap();
|
||||
assert_eq!(map.detect_layer_evidence(0).unwrap(), EvidenceKind::Exact);
|
||||
@@ -212,7 +212,7 @@ fn detect_layer_evidence_exact_vs_approx() {
|
||||
let dir2 = tempdir().unwrap();
|
||||
let layer0b = layer_dir(dir2.path(), 0);
|
||||
write_unitigs_at(&layer0b, &[b"AAAACGT"]);
|
||||
Layer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap();
|
||||
TypedLayer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap();
|
||||
PartitionMeta { n_layers: 1, mode: approx }.save(dir2.path()).unwrap();
|
||||
let map2 = LayeredMap::<()>::open(dir2.path()).unwrap();
|
||||
assert_eq!(map2.detect_layer_evidence(0).unwrap(), EvidenceKind::Approx);
|
||||
|
||||
Reference in New Issue
Block a user