Rename Layer<D> to TypedLayer and introduce heterogeneous Layer enum
Renames `Layer<D>` to `TypedLayer<D>` to establish a distinct typed abstraction. Introduces a new heterogeneous `Layer` enum that unifies count and presence storage with runtime dispatch, delegating operations to the underlying typed variants. Updates cache, index, and phylo consumers to align with the renamed type and new extension traits, preserving full test suite stability while preparing the foundation for multi-partition caching.
This commit is contained in:
@@ -133,14 +133,35 @@ needed — `obikindex → obikpartitionner` stays the same direction as before.
|
|||||||
Full workspace test suite green (0 failed) after, including all 27
|
Full workspace test suite green (0 failed) after, including all 27
|
||||||
`obikphylo::siblings` tests.
|
`obikphylo::siblings` tests.
|
||||||
|
|
||||||
Still open: (1)/(2) themselves — `AnyLayer` (or whatever it ends up named;
|
## (1) done (2026-08-20): `Layer` is now the heterogeneous handle, `Mat` is gone
|
||||||
`AnyLayer` was rejected as a placeholder, no replacement chosen yet) and
|
|
||||||
`KmerPartition` (singular, one partition's open layers) are not built yet.
|
Resolved the naming question left open above. `Layer<D>` (the old
|
||||||
`obikphylo::siblings::cache::{Mat, PartitionCache}` and
|
generic/monomorphic type) renamed to `TypedLayer<D>` throughout
|
||||||
`obikpartitionner::query_layer` (now `obikindex::query_layer`)'s
|
(`obilayeredmap`, `obikindex`, `obikphylo` — 12 files, mechanical) to free
|
||||||
`QueryLayer` still each independently bundle MPHF+matrix — unchanged by
|
`Layer` for the type that's actually meant to be everyone's default
|
||||||
this restructuring, which was purely about *where* code lives, not about
|
handle. `obilayeredmap::content_layer::Layer` (re-exported at the crate
|
||||||
building the heterogeneous-layer cache itself.
|
root) is that type — `Count(TypedLayer<PersistentCompactIntMatrix>)`/
|
||||||
|
`Presence(TypedLayer<PersistentBitMatrix>)`, `Layer::open` doing the same
|
||||||
|
disk probe `Mat::open` used to, `find_slot`/`index_batch`/`n_cols`/
|
||||||
|
`fill_sub_matrix_carries` dispatching per variant exactly as `Mat` did.
|
||||||
|
|
||||||
|
`obikphylo::siblings::cache::Mat` deleted outright — `PartitionCache` now
|
||||||
|
holds `Vec<Vec<obilayeredmap::Layer>>` directly. The one sibling-specific
|
||||||
|
method `Mat` carried (`iter_minorants_batch`) is not on `obilayeredmap::
|
||||||
|
Layer` (phylo concepts don't belong in `obilayeredmap`) — it's an
|
||||||
|
`impl SiblingLayerExt for obilayeredmap::Layer` in `iter.rs`, dispatching
|
||||||
|
to each variant's existing `impl<D: LayerData> SiblingLayerExt for
|
||||||
|
TypedLayer<D>`.
|
||||||
|
|
||||||
|
Full workspace suite green (0 failed) after, including all 27
|
||||||
|
`obikphylo::siblings` tests.
|
||||||
|
|
||||||
|
Still not built: (2) — `KmerPartition` (singular, one partition's open
|
||||||
|
`Vec<Layer>`) and a multi-partition cache in `obikpartitionner` to replace
|
||||||
|
`obikphylo::siblings::cache::PartitionCache` and `obikindex::query_layer`'s
|
||||||
|
still-separate `QueryLayer` (which still independently bundles MPHF+matrix,
|
||||||
|
2-way not using `Layer` at all). Both remaining consumers now sit one
|
||||||
|
`Layer::open` call away from unifying onto (2) once it exists.
|
||||||
|
|
||||||
## The problem
|
## The problem
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ use obipipeline::{
|
|||||||
|
|
||||||
use obidebruinj::GraphDeBruijn;
|
use obidebruinj::GraphDeBruijn;
|
||||||
use obikseq::CanonicalKmer;
|
use obikseq::CanonicalKmer;
|
||||||
use obilayeredmap::{IndexMode, Layer};
|
use obilayeredmap::{IndexMode, TypedLayer};
|
||||||
use obiskio::{SKError, SKResult};
|
use obiskio::{SKError, SKResult};
|
||||||
|
|
||||||
use crate::common::olm_to_sk;
|
use crate::common::olm_to_sk;
|
||||||
@@ -125,7 +125,7 @@ pub(crate) fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> SKRe
|
|||||||
let n_kmers = g.len();
|
let n_kmers = g.len();
|
||||||
g.compute_degrees_and_mark_starts();
|
g.compute_degrees_and_mark_starts();
|
||||||
std::fs::create_dir_all(layer_dir)?;
|
std::fs::create_dir_all(layer_dir)?;
|
||||||
let mut uw = Layer::<()>::unitig_writer(layer_dir).map_err(|e| olm_to_sk(e, "graph pipeline"))?;
|
let mut uw = TypedLayer::<()>::unitig_writer(layer_dir).map_err(|e| olm_to_sk(e, "graph pipeline"))?;
|
||||||
g.try_for_each_unitig(|unitig| uw.write(unitig))?;
|
g.try_for_each_unitig(|unitig| uw.write(unitig))?;
|
||||||
uw.close()?;
|
uw.close()?;
|
||||||
drop(g);
|
drop(g);
|
||||||
@@ -134,7 +134,7 @@ pub(crate) fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> SKRe
|
|||||||
|
|
||||||
// ── materialize_layer ─────────────────────────────────────────────────────────
|
// ── materialize_layer ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/// Phase 2 (full): write_graph_as_unitigs + `Layer::<()>::build`.
|
/// Phase 2 (full): write_graph_as_unitigs + `TypedLayer::<()>::build`.
|
||||||
///
|
///
|
||||||
/// Returns n_kmers.
|
/// Returns n_kmers.
|
||||||
pub(crate) fn materialize_layer(
|
pub(crate) fn materialize_layer(
|
||||||
@@ -145,7 +145,7 @@ pub(crate) fn materialize_layer(
|
|||||||
) -> SKResult<usize> {
|
) -> SKResult<usize> {
|
||||||
let n = write_graph_as_unitigs(g, layer_dir)?;
|
let n = write_graph_as_unitigs(g, layer_dir)?;
|
||||||
debug!("materialize_layer: unitigs written ({n} kmers), building MPHF");
|
debug!("materialize_layer: unitigs written ({n} kmers), building MPHF");
|
||||||
Layer::<()>::build(layer_dir, block_bits, evidence)
|
TypedLayer::<()>::build(layer_dir, block_bits, evidence)
|
||||||
.map_err(|e| olm_to_sk(e, "graph pipeline"))?;
|
.map_err(|e| olm_to_sk(e, "graph pipeline"))?;
|
||||||
debug!("materialize_layer: MPHF build done");
|
debug!("materialize_layer: MPHF build done");
|
||||||
Ok(n)
|
Ok(n)
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ use epserde::prelude::*;
|
|||||||
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
|
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
|
||||||
use obidebruinj::GraphDeBruijn;
|
use obidebruinj::GraphDeBruijn;
|
||||||
use obilayeredmap::meta::PartitionMeta;
|
use obilayeredmap::meta::PartitionMeta;
|
||||||
use obilayeredmap::{IndexMode, layer::Layer};
|
use obilayeredmap::{IndexMode, layer::TypedLayer};
|
||||||
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
||||||
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
||||||
|
|
||||||
@@ -96,7 +96,7 @@ impl KmerIndex {
|
|||||||
let n_kmers =
|
let n_kmers =
|
||||||
if with_counts {
|
if with_counts {
|
||||||
let n = write_graph_as_unitigs(g, &layer_dir)?;
|
let n = write_graph_as_unitigs(g, &layer_dir)?;
|
||||||
Layer::<PersistentCompactIntMatrix>::build(&layer_dir, block_bits, mode, |kmer| {
|
TypedLayer::<PersistentCompactIntMatrix>::build(&layer_dir, block_bits, mode, |kmer| {
|
||||||
match (&mphf1_opt, &counts1_opt) {
|
match (&mphf1_opt, &counts1_opt) {
|
||||||
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
|
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
|
||||||
_ => 1,
|
_ => 1,
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ use tracing::debug;
|
|||||||
|
|
||||||
use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder};
|
use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder};
|
||||||
use obikseq::CanonicalKmer;
|
use obikseq::CanonicalKmer;
|
||||||
use obilayeredmap::{IndexMode, Layer, LayeredMap, MphfOnly, layer_dir};
|
use obilayeredmap::{IndexMode, TypedLayer, LayeredMap, MphfOnly, layer_dir};
|
||||||
use obiskio::{SKError, SKResult, UnitigFileReader};
|
use obiskio::{SKError, SKResult, UnitigFileReader};
|
||||||
|
|
||||||
use crate::common::{ColBuilder, load_meta, olm_to_sk};
|
use crate::common::{ColBuilder, load_meta, olm_to_sk};
|
||||||
@@ -232,7 +232,7 @@ impl KmerIndex {
|
|||||||
// (all slots true — every kmer in those layers belongs to genome_0).
|
// (all slots true — every kmer in those layers belongs to genome_0).
|
||||||
if n_dst_genomes == 1 && mode == MergeMode::Presence {
|
if n_dst_genomes == 1 && mode == MergeMode::Presence {
|
||||||
for l in 0..n_dst_layers {
|
for l in 0..n_dst_layers {
|
||||||
Layer::<()>::init_presence_matrix(
|
TypedLayer::<()>::init_presence_matrix(
|
||||||
&layer_dir(&dst_index_dir, l),
|
&layer_dir(&dst_index_dir, l),
|
||||||
dst_map.layer(l).n(),
|
dst_map.layer(l).n(),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
use obilayeredmap::{IndexMode, layer::Layer};
|
use obilayeredmap::{IndexMode, layer::TypedLayer};
|
||||||
use obisys::{Reporter, Stage, progress_bar};
|
use obisys::{Reporter, Stage, progress_bar};
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
@@ -90,10 +90,10 @@ fn reindex_partition(
|
|||||||
fn reindex_layer(layer_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
|
fn reindex_layer(layer_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
|
||||||
match target {
|
match target {
|
||||||
IndexMode::Exact => {
|
IndexMode::Exact => {
|
||||||
Layer::<()>::build_exact_evidence(layer_dir, block_bits).map_err(olm_to_oki)?;
|
TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits).map_err(olm_to_oki)?;
|
||||||
}
|
}
|
||||||
IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => {
|
IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => {
|
||||||
Layer::<()>::build_approx_evidence(layer_dir, *b, *z).map_err(olm_to_oki)?;
|
TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z).map_err(olm_to_oki)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
remove_stale_evidence(layer_dir, target)
|
remove_stale_evidence(layer_dir, target)
|
||||||
|
|||||||
@@ -1,18 +1,11 @@
|
|||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
|
||||||
use obikseq::CanonicalKmer;
|
use obikseq::CanonicalKmer;
|
||||||
use obilayeredmap::meta::IndexMode;
|
use obilayeredmap::Layer;
|
||||||
use obilayeredmap::{Layer, OLMResult};
|
|
||||||
use obisys::progress_bar;
|
use obisys::progress_bar;
|
||||||
|
|
||||||
use obikindex::{KmerIndex, OKIResult};
|
use obikindex::{KmerIndex, OKIResult};
|
||||||
|
|
||||||
use super::SiblingAnnex;
|
|
||||||
use super::iter::SiblingLayerExt;
|
|
||||||
|
|
||||||
/// Every partition's already-open layers, built **once** for the whole
|
/// Every partition's already-open layers, built **once** for the whole
|
||||||
/// `build_sibling_annex` run and shared (read-only) across every lookup, in
|
/// `build_sibling_annex` run and shared (read-only) across every lookup, in
|
||||||
/// every source layer, for the rest of the run — not reopened/re-mmap'd per
|
/// every source layer, for the rest of the run — not reopened/re-mmap'd per
|
||||||
@@ -31,119 +24,14 @@ use super::iter::SiblingLayerExt;
|
|||||||
/// partition for the entire run, regardless of how many source layers or
|
/// partition for the entire run, regardless of how many source layers or
|
||||||
/// lookups follow.
|
/// lookups follow.
|
||||||
///
|
///
|
||||||
/// One `Layer<D>` per layer (MPHF + matrix bundled), not a separate
|
/// `mats[partition][layer]` holds `obilayeredmap::Layer` — the
|
||||||
/// `MphfLayer` and a separate `PersistentCompactIntMatrix`/
|
/// format-erased `Count`/`Presence` handle, not a sibling-specific type:
|
||||||
/// `PersistentBitMatrix` in parallel arrays — `obilayeredmap::Layer` already
|
/// this module used to bundle its own `Mat` enum here, duplicating exactly
|
||||||
/// *is* that bundle, with `find_slot` (MPHF-only, no data read),
|
/// what `Layer` now does one crate down (see
|
||||||
/// `n_cols`/`sub_matrix`/`fill_sub_matrix` (batched, sorted-internally
|
/// `DevDocMD/implementation/partition_layer_cache.md`). Its
|
||||||
/// column access) on top of it. Reinventing that pairing here would just be
|
/// sibling-specific extension (`iter_minorants_batch`) lives in `iter.rs`.
|
||||||
/// going back through the low-level pieces `Layer` already assembles.
|
|
||||||
pub(super) enum Mat {
|
|
||||||
Count(Layer<PersistentCompactIntMatrix>),
|
|
||||||
/// Dense *or* sparse (`pack --sparse`d) presence — `PersistentBitMatrix`
|
|
||||||
/// itself is a 4-way enum (`Columnar`/`Packed`/`Sparse`/`Implicit`) that
|
|
||||||
/// already auto-detects sparse storage in its own `open()` (checks
|
|
||||||
/// `presence/sparse_meta.json`) and dispatches every method
|
|
||||||
/// (`row`/`fill_row`/`nonzero_iter`/...) across all four internally —
|
|
||||||
/// see `DevDocMD/implementation/partition_layer_cache.md`. A separate
|
|
||||||
/// `SparsePresence(Layer<PersistentSparseBitMatrix>)` arm used to exist
|
|
||||||
/// here, opened via its own `presence/is_multi.prsb` check; it
|
|
||||||
/// predated `PersistentBitMatrix` growing native sparse support and
|
|
||||||
/// was pure duplication by the time it was removed — every method
|
|
||||||
/// below dispatched it identically to this arm.
|
|
||||||
Presence(Layer<PersistentBitMatrix>),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Mat {
|
|
||||||
/// Open one layer's matrix, auto-detecting count vs. presence from what
|
|
||||||
/// is actually on disk — the single source of truth for *every* caller
|
|
||||||
/// that opens a layer's own matrix (this module's cross-partition
|
|
||||||
/// [`PartitionCache::build`] and `family_scan::scan_layer_families`'s
|
|
||||||
/// own-layer lookup alike), so the two can never disagree. Sparse vs.
|
|
||||||
/// dense presence storage is `PersistentBitMatrix::open`'s own concern
|
|
||||||
/// (see the [`Presence`](Mat::Presence) variant's docs), not decided
|
|
||||||
/// here.
|
|
||||||
pub(super) fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult<Self> {
|
|
||||||
if with_counts && layer_dir.join("counts").exists() {
|
|
||||||
return Layer::<PersistentCompactIntMatrix>::open(layer_dir, mode).map(Mat::Count);
|
|
||||||
}
|
|
||||||
Layer::<PersistentBitMatrix>::open(layer_dir, mode).map(Mat::Presence)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
|
|
||||||
match self {
|
|
||||||
Mat::Count(l) => l.find_slot(kmer),
|
|
||||||
Mat::Presence(l) => l.find_slot(kmer),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Raw MPHF batch lookup: kmer → slot, no membership check — for
|
|
||||||
/// callers that already know every kmer is a member of *this* layer
|
|
||||||
/// (e.g. it came from this layer's own `iter_minorants_batch`), so the
|
|
||||||
/// evidence check `find_slot`/`find` would perform is redundant work.
|
|
||||||
/// See `DevDocMD/architecture/siblings.md`: iteration-pipeline kmers use
|
|
||||||
/// `index`, never `find`.
|
|
||||||
pub(super) fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
|
|
||||||
match self {
|
|
||||||
Mat::Count(l) => l.index_batch(kmers),
|
|
||||||
Mat::Presence(l) => l.index_batch(kmers),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// This layer's own `SiblingLayerExt::iter_minorants_batch` — dispatch
|
|
||||||
/// only, both arms return the same concrete `MinorantBatchIter` (it
|
|
||||||
/// doesn't depend on `D`), so no boxing is needed.
|
|
||||||
pub(super) fn iter_minorants_batch(
|
|
||||||
&self,
|
|
||||||
annex: std::sync::Arc<SiblingAnnex>,
|
|
||||||
batch_size: usize,
|
|
||||||
) -> super::iter::MinorantBatchIter {
|
|
||||||
match self {
|
|
||||||
Mat::Count(l) => l.iter_minorants_batch(annex, batch_size),
|
|
||||||
Mat::Presence(l) => l.iter_minorants_batch(annex, batch_size),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn n_cols(&self) -> usize {
|
|
||||||
match self {
|
|
||||||
Mat::Count(l) => l.n_cols(),
|
|
||||||
Mat::Presence(l) => l.n_cols(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` =
|
|
||||||
/// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must
|
|
||||||
/// have one entry per genome column, each resized to `slots.len()`.
|
|
||||||
///
|
|
||||||
/// Delegates entirely to `Layer<D>::fill_sub_matrix`, which sorts
|
|
||||||
/// `slots` once internally for a sequential mmap sweep per column, then
|
|
||||||
/// restores the original order — the same discipline this call site
|
|
||||||
/// (and `find_presence_batch`) used to hand-roll with its own sort +
|
|
||||||
/// genome-major loop. `Count` still needs one intermediate
|
|
||||||
/// `Vec<Vec<u32>>` fetch (the underlying store only has an int
|
|
||||||
/// sub-matrix, not a bool one), converted to presence (`!= 0`) in place
|
|
||||||
/// — the sort/sequential-access win is unaffected, just one extra
|
|
||||||
/// allocation pass over already-in-hand data.
|
|
||||||
pub(super) fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec<bool>]) {
|
|
||||||
match self {
|
|
||||||
Mat::Presence(l) => l.fill_sub_matrix(slots, out),
|
|
||||||
Mat::Count(l) => {
|
|
||||||
let mut counts: Vec<Vec<u32>> = out.iter().map(|_| Vec::new()).collect();
|
|
||||||
l.fill_sub_matrix(slots, &mut counts);
|
|
||||||
for (o, c) in out.iter_mut().zip(counts.iter()) {
|
|
||||||
o.clear();
|
|
||||||
o.extend(c.iter().map(|&v| v != 0));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) struct PartitionCache {
|
pub(super) struct PartitionCache {
|
||||||
/// `mats[partition][layer]` = that partition's opened layers. Used by
|
mats: Vec<Vec<Layer>>,
|
||||||
/// both [`obikindex::KmerIndex::build_sibling_annex`] and
|
|
||||||
/// [`obikindex::KmerIndex::sibling_annex_stats`].
|
|
||||||
mats: Vec<Vec<Mat>>,
|
|
||||||
/// Whether every `FamilyMask` field in this index's sibling annexes can
|
/// Whether every `FamilyMask` field in this index's sibling annexes can
|
||||||
/// be trusted as a real layer index (`n_layers <= 7`, the same decision
|
/// be trusted as a real layer index (`n_layers <= 7`, the same decision
|
||||||
/// `build_layer_sibling_annex` makes when writing them) — computed once
|
/// `build_layer_sibling_annex` makes when writing them) — computed once
|
||||||
@@ -163,9 +51,9 @@ impl PartitionCache {
|
|||||||
with_counts: bool,
|
with_counts: bool,
|
||||||
) -> OKIResult<Self> {
|
) -> OKIResult<Self> {
|
||||||
let pb = progress_bar("open_partitions", n_parts as u64, "partitions");
|
let pb = progress_bar("open_partitions", n_parts as u64, "partitions");
|
||||||
let built: Vec<(Vec<Mat>, usize)> = (0..n_parts)
|
let built: Vec<(Vec<Layer>, usize)> = (0..n_parts)
|
||||||
.into_par_iter()
|
.into_par_iter()
|
||||||
.map(|part| -> OKIResult<(Vec<Mat>, usize)> {
|
.map(|part| -> OKIResult<(Vec<Layer>, usize)> {
|
||||||
let index_dir = index.index_dir(part);
|
let index_dir = index.index_dir(part);
|
||||||
if !index_dir.exists() {
|
if !index_dir.exists() {
|
||||||
pb.inc(1);
|
pb.inc(1);
|
||||||
@@ -174,7 +62,7 @@ impl PartitionCache {
|
|||||||
let meta = index.partition_meta(part)?;
|
let meta = index.partition_meta(part)?;
|
||||||
let mut mats = Vec::with_capacity(meta.n_layers);
|
let mut mats = Vec::with_capacity(meta.n_layers);
|
||||||
for l in 0..meta.n_layers {
|
for l in 0..meta.n_layers {
|
||||||
let Ok(mat) = Mat::open(&index.layer_dir(part, l), &meta.mode, with_counts)
|
let Ok(mat) = Layer::open(&index.layer_dir(part, l), &meta.mode, with_counts)
|
||||||
else {
|
else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
@@ -331,7 +219,7 @@ impl PartitionCache {
|
|||||||
/// `hits` gets its slots (evidence-probed vs. trusted `index_batch`) — this
|
/// `hits` gets its slots (evidence-probed vs. trusted `index_batch`) — this
|
||||||
/// is everything after that.
|
/// is everything after that.
|
||||||
fn resolve_layer_hits(
|
fn resolve_layer_hits(
|
||||||
mat: &Mat,
|
mat: &Layer,
|
||||||
hits: &[(usize, usize, u8)],
|
hits: &[(usize, usize, u8)],
|
||||||
n_genomes: usize,
|
n_genomes: usize,
|
||||||
on_hit: &mut impl FnMut(usize, u8, usize),
|
on_hit: &mut impl FnMut(usize, u8, usize),
|
||||||
|
|||||||
@@ -60,9 +60,11 @@ use obipipeline::{ThrottleGuard, throttle};
|
|||||||
use obikindex::{OKIError, OKIResult};
|
use obikindex::{OKIError, OKIResult};
|
||||||
use obikindex::KmerIndex;
|
use obikindex::KmerIndex;
|
||||||
|
|
||||||
use super::cache::{Mat, PartitionCache};
|
use obilayeredmap::Layer;
|
||||||
|
|
||||||
|
use super::cache::PartitionCache;
|
||||||
use super::helpers::central_base;
|
use super::helpers::central_base;
|
||||||
use super::iter::SiblingEntry;
|
use super::iter::{SiblingEntry, SiblingLayerExt};
|
||||||
use super::{olm_to_ok, FamilyMask, SiblingAnnex, ANNEX_FILE_NAME};
|
use super::{olm_to_ok, FamilyMask, SiblingAnnex, ANNEX_FILE_NAME};
|
||||||
|
|
||||||
/// Families per batch — see the module docs for the memory-vs-per-partition-
|
/// Families per batch — see the module docs for the memory-vs-per-partition-
|
||||||
@@ -129,10 +131,10 @@ pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult<Vec<PathBuf>> {
|
|||||||
/// generating this layer's batches — opened once, not per batch. No `cache`
|
/// generating this layer's batches — opened once, not per batch. No `cache`
|
||||||
/// here: generation never touches the cross-partition cache, only this
|
/// here: generation never touches the cross-partition cache, only this
|
||||||
/// layer's own already-open matrix. No separate MPHF/`slot_kmer` either —
|
/// layer's own already-open matrix. No separate MPHF/`slot_kmer` either —
|
||||||
/// `mat` (a `Layer<D>`) already bundles the MPHF, and each `SiblingEntry`
|
/// `mat` (a `TypedLayer<D>`) already bundles the MPHF, and each `SiblingEntry`
|
||||||
/// arrives with its kmer and mask already in hand from `iter_minorants_batch`.
|
/// arrives with its kmer and mask already in hand from `iter_minorants_batch`.
|
||||||
struct LayerCtx {
|
struct LayerCtx {
|
||||||
mat: Mat,
|
mat: Layer,
|
||||||
n_parts: usize,
|
n_parts: usize,
|
||||||
n_genomes: usize,
|
n_genomes: usize,
|
||||||
n_cols: usize,
|
n_cols: usize,
|
||||||
@@ -193,7 +195,7 @@ pub(super) fn scan_layer_families(
|
|||||||
let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?;
|
let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?;
|
||||||
let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?);
|
let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?);
|
||||||
|
|
||||||
let mat = Mat::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?;
|
let mat = Layer::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?;
|
||||||
let n_cols = mat.n_cols().min(n_genomes);
|
let n_cols = mat.n_cols().min(n_genomes);
|
||||||
|
|
||||||
let ctx = Arc::new(LayerCtx { mat, n_parts, n_genomes, n_cols, k });
|
let ctx = Arc::new(LayerCtx { mat, n_parts, n_genomes, n_cols, k });
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
//! Phylo/sibling-domain iteration over a layer — an extension trait, not a
|
//! Phylo/sibling-domain iteration over a layer — an extension trait, not a
|
||||||
//! new field on `MphfLayer`/`Layer<D>`: "family"/"minorant" are phylo
|
//! new field on `MphfLayer`/`TypedLayer<D>`: "family"/"minorant" are phylo
|
||||||
//! concepts, `obilayeredmap` stays kmer/slot-mapping only (see
|
//! concepts, `obilayeredmap` stays kmer/slot-mapping only (see
|
||||||
//! `DevDocMD/architecture/siblings.md`).
|
//! `DevDocMD/architecture/siblings.md`).
|
||||||
//!
|
//!
|
||||||
@@ -29,7 +29,7 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use obikseq::CanonicalKmer;
|
use obikseq::CanonicalKmer;
|
||||||
use obilayeredmap::{KmerIter, Layer, LayerData};
|
use obilayeredmap::{KmerIter, TypedLayer, LayerData};
|
||||||
|
|
||||||
use super::{FamilyMask, SiblingAnnex};
|
use super::{FamilyMask, SiblingAnnex};
|
||||||
|
|
||||||
@@ -128,10 +128,10 @@ fn collect_batch<I: Iterator>(inner: &mut I, batch_size: usize) -> Option<Vec<I:
|
|||||||
if batch.is_empty() { None } else { Some(batch) }
|
if batch.is_empty() { None } else { Some(batch) }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Adds phylo/sibling iteration to any `Layer<D>` — the extension that
|
/// Adds phylo/sibling iteration to any `TypedLayer<D>` — the extension that
|
||||||
/// turns a plain layer into a "sibling layer". Generic over `D`
|
/// turns a plain layer into a "sibling layer". Generic over `D`
|
||||||
/// (`LayerData`) rather than implemented once per matrix kind: kmer
|
/// (`LayerData`) rather than implemented once per matrix kind: kmer
|
||||||
/// iteration doesn't depend on the data payload, and `Layer<D>` already
|
/// iteration doesn't depend on the data payload, and `TypedLayer<D>` already
|
||||||
/// delegates `iter_kmers`/`index`/`index_batch` to its inner MPHF for every
|
/// delegates `iter_kmers`/`index`/`index_batch` to its inner MPHF for every
|
||||||
/// `D` — reusing that instead of going back through a separate `MphfLayer`.
|
/// `D` — reusing that instead of going back through a separate `MphfLayer`.
|
||||||
pub trait SiblingLayerExt {
|
pub trait SiblingLayerExt {
|
||||||
@@ -154,7 +154,7 @@ pub trait SiblingLayerExt {
|
|||||||
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter;
|
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter;
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<D: LayerData> SiblingLayerExt for Layer<D> {
|
impl<D: LayerData> SiblingLayerExt for TypedLayer<D> {
|
||||||
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
|
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
|
||||||
SiblingIter { kmers: self.iter_kmers(), annex, order: 0 }
|
SiblingIter { kmers: self.iter_kmers(), annex, order: 0 }
|
||||||
}
|
}
|
||||||
@@ -171,3 +171,39 @@ impl<D: LayerData> SiblingLayerExt for Layer<D> {
|
|||||||
MinorantBatchIter { inner: self.iter_minorants(annex), batch_size }
|
MinorantBatchIter { inner: self.iter_minorants(annex), batch_size }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Same extension, over `obilayeredmap::Layer` (the format-erased
|
||||||
|
/// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch
|
||||||
|
/// only, both arms return the same concrete iterator types (they don't
|
||||||
|
/// depend on which `D` is inside), so no boxing is needed. `obilayeredmap`
|
||||||
|
/// itself can't implement this: "family"/"minorant" are phylo concepts, it
|
||||||
|
/// stays kmer/slot-mapping only (see the module docs above).
|
||||||
|
impl SiblingLayerExt for obilayeredmap::Layer {
|
||||||
|
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
|
||||||
|
match self {
|
||||||
|
obilayeredmap::Layer::Count(l) => l.iter_siblings(annex),
|
||||||
|
obilayeredmap::Layer::Presence(l) => l.iter_siblings(annex),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn iter_siblings_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> SiblingBatchIter {
|
||||||
|
match self {
|
||||||
|
obilayeredmap::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size),
|
||||||
|
obilayeredmap::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn iter_minorants(&self, annex: Arc<SiblingAnnex>) -> MinorantIter {
|
||||||
|
match self {
|
||||||
|
obilayeredmap::Layer::Count(l) => l.iter_minorants(annex),
|
||||||
|
obilayeredmap::Layer::Presence(l) => l.iter_minorants(annex),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter {
|
||||||
|
match self {
|
||||||
|
obilayeredmap::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size),
|
||||||
|
obilayeredmap::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -156,9 +156,9 @@ fn sibling_annex_works_after_pack_sparse() {
|
|||||||
// `pack --sparse` (`KmerIndex::pack_matrices(true)`) run on the merged
|
// `pack --sparse` (`KmerIndex::pack_matrices(true)`) run on the merged
|
||||||
// index before building the sibling annex — proves `PartitionCache`'s
|
// index before building the sibling annex — proves `PartitionCache`'s
|
||||||
// sparse-detection (`Mat::SparsePresence`, gated on `presence/
|
// sparse-detection (`Mat::SparsePresence`, gated on `presence/
|
||||||
// is_multi.prsb`) and the generic `Layer<D>` methods it relies on
|
// is_multi.prsb`) and the generic `TypedLayer<D>` methods it relies on
|
||||||
// actually round-trip through the real build pipeline, not just the
|
// actually round-trip through the real build pipeline, not just the
|
||||||
// unit-level `Layer<PersistentSparseBitMatrix>` tests in
|
// unit-level `TypedLayer<PersistentSparseBitMatrix>` tests in
|
||||||
// `obilayeredmap`.
|
// `obilayeredmap`.
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||||
|
|||||||
@@ -0,0 +1,113 @@
|
|||||||
|
//! [`Layer`] — the format-erased layer handle every caller outside this
|
||||||
|
//! crate should reach for. `TypedLayer<D>` (`layer.rs`) is monomorphic: a
|
||||||
|
//! `Vec<TypedLayer<D>>` needs one `D` fixed at compile time for every
|
||||||
|
//! element, which real partitions don't respect (`with_counts` can differ
|
||||||
|
//! layer to layer across merges; see
|
||||||
|
//! `DevDocMD/implementation/partition_layer_cache.md`). `Layer` is the
|
||||||
|
//! sum type that lets a partition's layers be held together regardless —
|
||||||
|
//! `Count`/`Presence` each wrap the one concrete `TypedLayer<D>` that
|
||||||
|
//! content implies, chosen by [`Layer::open`]'s own disk probe.
|
||||||
|
//!
|
||||||
|
//! This is exactly the shape `obikphylo::siblings::cache::Mat` used to
|
||||||
|
//! reimplement locally, minus its one sibling-specific method
|
||||||
|
//! (`iter_minorants_batch`, which stays an extension trait over there —
|
||||||
|
//! `obilayeredmap` has no business knowing about sibling annexes).
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
||||||
|
use obikseq::CanonicalKmer;
|
||||||
|
|
||||||
|
use crate::error::OLMResult;
|
||||||
|
use crate::layer::{LayerContent, TypedLayer, COUNTS_DIR};
|
||||||
|
use crate::meta::IndexMode;
|
||||||
|
use crate::mphf_layer::EvidenceKind;
|
||||||
|
|
||||||
|
/// One layer, its content (`Count`/`Presence`) resolved at open time rather
|
||||||
|
/// than at compile time — see the module docs.
|
||||||
|
pub enum Layer {
|
||||||
|
Count(TypedLayer<PersistentCompactIntMatrix>),
|
||||||
|
Presence(TypedLayer<PersistentBitMatrix>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Layer {
|
||||||
|
/// Open one layer, auto-detecting count vs. presence from what is
|
||||||
|
/// actually on disk (`counts/` present and wanted, else presence) — the
|
||||||
|
/// single source of truth every caller that opens a layer's own matrix
|
||||||
|
/// should share, so two callers can never disagree about which format a
|
||||||
|
/// given layer is. Dense vs. sparse presence storage is
|
||||||
|
/// `PersistentBitMatrix::open`'s own concern (it's a 4-way
|
||||||
|
/// `Columnar`/`Packed`/`Sparse`/`Implicit` enum internally), not decided
|
||||||
|
/// here.
|
||||||
|
pub fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult<Self> {
|
||||||
|
if with_counts && layer_dir.join(COUNTS_DIR).exists() {
|
||||||
|
return TypedLayer::<PersistentCompactIntMatrix>::open(layer_dir, mode).map(Layer::Count);
|
||||||
|
}
|
||||||
|
TypedLayer::<PersistentBitMatrix>::open(layer_dir, mode).map(Layer::Presence)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn content(&self) -> LayerContent {
|
||||||
|
match self {
|
||||||
|
Layer::Count(_) => LayerContent::Count,
|
||||||
|
Layer::Presence(_) => LayerContent::Presence,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn evidence_kind(&self) -> EvidenceKind {
|
||||||
|
match self {
|
||||||
|
Layer::Count(l) => l.evidence_kind(),
|
||||||
|
Layer::Presence(l) => l.evidence_kind(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn n(&self) -> usize {
|
||||||
|
match self {
|
||||||
|
Layer::Count(l) => l.n(),
|
||||||
|
Layer::Presence(l) => l.n(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
|
||||||
|
match self {
|
||||||
|
Layer::Count(l) => l.find_slot(kmer),
|
||||||
|
Layer::Presence(l) => l.find_slot(kmer),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Raw MPHF batch lookup: kmer → slot, no membership check — for
|
||||||
|
/// callers that already know every kmer is a member of *this* layer, so
|
||||||
|
/// the evidence check `find_slot`/`find` would perform is redundant.
|
||||||
|
pub fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
|
||||||
|
match self {
|
||||||
|
Layer::Count(l) => l.index_batch(kmers),
|
||||||
|
Layer::Presence(l) => l.index_batch(kmers),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn n_cols(&self) -> usize {
|
||||||
|
match self {
|
||||||
|
Layer::Count(l) => l.n_cols(),
|
||||||
|
Layer::Presence(l) => l.n_cols(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` =
|
||||||
|
/// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must
|
||||||
|
/// have one entry per genome column, each resized to `slots.len()`.
|
||||||
|
/// `Count` needs one intermediate `Vec<Vec<u32>>` fetch (the underlying
|
||||||
|
/// store only has an int sub-matrix, not a bool one), converted to
|
||||||
|
/// presence (`!= 0`) in place.
|
||||||
|
pub fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec<bool>]) {
|
||||||
|
match self {
|
||||||
|
Layer::Presence(l) => l.fill_sub_matrix(slots, out),
|
||||||
|
Layer::Count(l) => {
|
||||||
|
let mut counts: Vec<Vec<u32>> = out.iter().map(|_| Vec::new()).collect();
|
||||||
|
l.fill_sub_matrix(slots, &mut counts);
|
||||||
|
for (o, c) in out.iter_mut().zip(counts.iter()) {
|
||||||
|
o.clear();
|
||||||
|
o.extend(c.iter().map(|&v| v != 0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -27,7 +27,7 @@ pub trait LayerData: Sized {
|
|||||||
fn read(&self, slot: usize) -> Self::Item;
|
fn read(&self, slot: usize) -> Self::Item;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Layer directory path for layer `i` within a partition's index root — the
|
/// TypedLayer directory path for layer `i` within a partition's index root — the
|
||||||
/// single source of truth for the on-disk `layer_N` naming convention
|
/// single source of truth for the on-disk `layer_N` naming convention
|
||||||
/// (mirrors what `LayeredMap::open`/`push_layer` already use internally).
|
/// (mirrors what `LayeredMap::open`/`push_layer` already use internally).
|
||||||
///
|
///
|
||||||
@@ -43,7 +43,7 @@ pub fn layer_dir(root: &Path, i: usize) -> PathBuf {
|
|||||||
/// Opens layer `i`'s data only, skipping the MPHF — for callers that only
|
/// Opens layer `i`'s data only, skipping the MPHF — for callers that only
|
||||||
/// need matrix-level operations (distance traits, column weights, group
|
/// need matrix-level operations (distance traits, column weights, group
|
||||||
/// filters, sub-matrix extraction) and never look up a kmer for this layer.
|
/// filters, sub-matrix extraction) and never look up a kmer for this layer.
|
||||||
/// `Layer<D>::open` always pays for the MPHF too (and doesn't expose `data`
|
/// `TypedLayer<D>::open` always pays for the MPHF too (and doesn't expose `data`
|
||||||
/// once open), so it's the wrong tool for these; this is the other half of
|
/// once open), so it's the wrong tool for these; this is the other half of
|
||||||
/// the same `D::open(layer_dir)` call, without the MPHF alongside it.
|
/// the same `D::open(layer_dir)` call, without the MPHF alongside it.
|
||||||
///
|
///
|
||||||
@@ -89,7 +89,7 @@ impl LayerData for PersistentSparseBitMatrix {
|
|||||||
|
|
||||||
/// What a layer's data matrix represents — orthogonal to *how* it's stored
|
/// What a layer's data matrix represents — orthogonal to *how* it's stored
|
||||||
/// on disk (`obicompactvec::StorageKind`'s concern). Only meaningful for
|
/// on disk (`obicompactvec::StorageKind`'s concern). Only meaningful for
|
||||||
/// `D` that actually carry a matrix: `Layer<()>` (mode 1, set membership,
|
/// `D` that actually carry a matrix: `TypedLayer<()>` (mode 1, set membership,
|
||||||
/// no matrix at all) has neither a `LayerContent` nor a `content()` method
|
/// no matrix at all) has neither a `LayerContent` nor a `content()` method
|
||||||
/// — it's a write-time-only state, never a queryable content. Once a layer
|
/// — it's a write-time-only state, never a queryable content. Once a layer
|
||||||
/// is closed, "no matrix file" reads back as `Presence` via
|
/// is closed, "no matrix file" reads back as `Presence` via
|
||||||
@@ -115,7 +115,7 @@ impl LayerContent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Implemented only by `D` that carry a real matrix — gates
|
/// Implemented only by `D` that carry a real matrix — gates
|
||||||
/// [`Layer::content`](Layer::content) so `Layer<()>` doesn't get it.
|
/// [`TypedLayer::content`](TypedLayer::content) so `TypedLayer<()>` doesn't get it.
|
||||||
pub trait HasLayerContent {
|
pub trait HasLayerContent {
|
||||||
const CONTENT: LayerContent;
|
const CONTENT: LayerContent;
|
||||||
}
|
}
|
||||||
@@ -135,7 +135,7 @@ impl HasLayerContent for PersistentSparseBitMatrix {
|
|||||||
// ── StorageKind passthrough ────────────────────────────────────────────────────
|
// ── StorageKind passthrough ────────────────────────────────────────────────────
|
||||||
|
|
||||||
/// Implemented only by `D` that expose their own `storage_kind()` — gates
|
/// Implemented only by `D` that expose their own `storage_kind()` — gates
|
||||||
/// [`Layer::storage_kind`]. `PersistentSparseBitMatrix` has a single,
|
/// [`TypedLayer::storage_kind`]. `PersistentSparseBitMatrix` has a single,
|
||||||
/// fixed on-disk format (no `Columnar`/`Packed`/`Implicit` variants of its
|
/// fixed on-disk format (no `Columnar`/`Packed`/`Implicit` variants of its
|
||||||
/// own), so it's deliberately not given an impl: there's nothing to report
|
/// own), so it's deliberately not given an impl: there's nothing to report
|
||||||
/// beyond what `content()` already says.
|
/// beyond what `content()` already says.
|
||||||
@@ -157,7 +157,7 @@ impl HasStorageKind for PersistentBitMatrix {
|
|||||||
|
|
||||||
// ── Structures ────────────────────────────────────────────────────────────────
|
// ── Structures ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
pub struct Layer<D: LayerData = ()> {
|
pub struct TypedLayer<D: LayerData = ()> {
|
||||||
mphf: MphfLayer,
|
mphf: MphfLayer,
|
||||||
data: D,
|
data: D,
|
||||||
}
|
}
|
||||||
@@ -169,7 +169,7 @@ pub struct Hit<T = ()> {
|
|||||||
|
|
||||||
// ── Common read path ──────────────────────────────────────────────────────────
|
// ── Common read path ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
impl<D: LayerData> Layer<D> {
|
impl<D: LayerData> TypedLayer<D> {
|
||||||
pub fn open(path: &Path, mode: &IndexMode) -> OLMResult<Self> {
|
pub fn open(path: &Path, mode: &IndexMode) -> OLMResult<Self> {
|
||||||
let mphf = MphfLayer::open(path, mode)?;
|
let mphf = MphfLayer::open(path, mode)?;
|
||||||
let data = D::open(path)?;
|
let data = D::open(path)?;
|
||||||
@@ -249,7 +249,7 @@ impl<D: LayerData> Layer<D> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<D: LayerData + HasLayerContent> Layer<D> {
|
impl<D: LayerData + HasLayerContent> TypedLayer<D> {
|
||||||
/// This already-open layer's content (`Count`/`Presence`) — a
|
/// This already-open layer's content (`Count`/`Presence`) — a
|
||||||
/// compile-time fact about `D`, not a runtime read.
|
/// compile-time fact about `D`, not a runtime read.
|
||||||
pub const fn content(&self) -> LayerContent {
|
pub const fn content(&self) -> LayerContent {
|
||||||
@@ -257,7 +257,7 @@ impl<D: LayerData + HasLayerContent> Layer<D> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<D: LayerData + HasStorageKind> Layer<D> {
|
impl<D: LayerData + HasStorageKind> TypedLayer<D> {
|
||||||
/// This already-open layer's storage format — delegates to `D`'s own
|
/// This already-open layer's storage format — delegates to `D`'s own
|
||||||
/// `storage_kind()`, reading a discriminant already in memory.
|
/// `storage_kind()`, reading a discriminant already in memory.
|
||||||
pub fn storage_kind(&self) -> obicompactvec::StorageKind {
|
pub fn storage_kind(&self) -> obicompactvec::StorageKind {
|
||||||
@@ -267,7 +267,7 @@ impl<D: LayerData + HasStorageKind> Layer<D> {
|
|||||||
|
|
||||||
// ── Mode 1 — set membership ───────────────────────────────────────────────────
|
// ── Mode 1 — set membership ───────────────────────────────────────────────────
|
||||||
|
|
||||||
impl Layer<()> {
|
impl TypedLayer<()> {
|
||||||
pub fn build(out_dir: &Path, block_bits: u8, mode: &IndexMode) -> OLMResult<usize> {
|
pub fn build(out_dir: &Path, block_bits: u8, mode: &IndexMode) -> OLMResult<usize> {
|
||||||
MphfLayer::build(out_dir, block_bits, mode, &mut |_, _| Ok(()))
|
MphfLayer::build(out_dir, block_bits, mode, &mut |_, _| Ok(()))
|
||||||
}
|
}
|
||||||
@@ -284,7 +284,7 @@ impl Layer<()> {
|
|||||||
|
|
||||||
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
|
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
impl Layer<PersistentCompactIntMatrix> {
|
impl TypedLayer<PersistentCompactIntMatrix> {
|
||||||
pub fn build(
|
pub fn build(
|
||||||
out_dir: &Path,
|
out_dir: &Path,
|
||||||
block_bits: u8,
|
block_bits: u8,
|
||||||
@@ -317,7 +317,7 @@ impl Layer<PersistentCompactIntMatrix> {
|
|||||||
|
|
||||||
// ── Mode 2 — count matrix column append ──────────────────────────────────────
|
// ── Mode 2 — count matrix column append ──────────────────────────────────────
|
||||||
|
|
||||||
impl Layer<PersistentCompactIntMatrix> {
|
impl TypedLayer<PersistentCompactIntMatrix> {
|
||||||
pub fn append_genome_column(
|
pub fn append_genome_column(
|
||||||
layer_dir: &Path,
|
layer_dir: &Path,
|
||||||
value_of: impl Fn(usize) -> u32,
|
value_of: impl Fn(usize) -> u32,
|
||||||
@@ -357,14 +357,14 @@ impl Layer<PersistentCompactIntMatrix> {
|
|||||||
//
|
//
|
||||||
// `n_cols`/`sub_matrix`/`fill_sub_matrix` are identical in shape for any
|
// `n_cols`/`sub_matrix`/`fill_sub_matrix` are identical in shape for any
|
||||||
// `D` satisfying `BinaryMatrix` — genuinely generic (not just two
|
// `D` satisfying `BinaryMatrix` — genuinely generic (not just two
|
||||||
// near-identical impl blocks) so `Layer<PersistentSparseBitMatrix>` gets
|
// near-identical impl blocks) so `TypedLayer<PersistentSparseBitMatrix>` gets
|
||||||
// them for free, matching `Layer<PersistentBitMatrix>` at the same call
|
// them for free, matching `TypedLayer<PersistentBitMatrix>` at the same call
|
||||||
// sites (see `obikphylo::siblings::cache::Mat`). Construction
|
// sites (see `obikphylo::siblings::cache::Mat`). Construction
|
||||||
// (`append_genome_column`/`build_presence`) stays `PersistentBitMatrix`-only
|
// (`append_genome_column`/`build_presence`) stays `PersistentBitMatrix`-only
|
||||||
// below: sparse matrices aren't built column-by-column, they're built
|
// below: sparse matrices aren't built column-by-column, they're built
|
||||||
// row-by-row from an already-built dense layer
|
// row-by-row from an already-built dense layer
|
||||||
// (`PersistentSparseBitMatrixBuilder::build_from_dense`).
|
// (`PersistentSparseBitMatrixBuilder::build_from_dense`).
|
||||||
impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix> Layer<D> {
|
impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix> TypedLayer<D> {
|
||||||
/// Number of genome columns in this layer's presence matrix — see
|
/// Number of genome columns in this layer's presence matrix — see
|
||||||
/// `PersistentBitMatrix::n_cols`'s docs for the `Implicit` mono-genome
|
/// `PersistentBitMatrix::n_cols`'s docs for the `Implicit` mono-genome
|
||||||
/// special case (always reports `1`, regardless of the index's real
|
/// special case (always reports `1`, regardless of the index's real
|
||||||
@@ -395,7 +395,7 @@ impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix> Layer<D> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Layer<PersistentBitMatrix> {
|
impl TypedLayer<PersistentBitMatrix> {
|
||||||
pub fn append_genome_column(
|
pub fn append_genome_column(
|
||||||
layer_dir: &Path,
|
layer_dir: &Path,
|
||||||
value_of: impl Fn(usize) -> bool,
|
value_of: impl Fn(usize) -> bool,
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
pub mod content_layer;
|
||||||
pub mod error;
|
pub mod error;
|
||||||
pub mod evidence;
|
pub mod evidence;
|
||||||
pub mod fingerprint;
|
pub mod fingerprint;
|
||||||
@@ -7,8 +8,9 @@ pub mod map;
|
|||||||
pub mod meta;
|
pub mod meta;
|
||||||
pub(crate) mod mphf_layer;
|
pub(crate) mod mphf_layer;
|
||||||
|
|
||||||
|
pub use content_layer::Layer;
|
||||||
pub use error::{OLMError, OLMResult};
|
pub use error::{OLMError, OLMResult};
|
||||||
pub use layer::{layer_dir, open_data, HasLayerContent, HasStorageKind, Hit, Layer, LayerContent, LayerData};
|
pub use layer::{layer_dir, open_data, HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer};
|
||||||
pub use layered_store::LayeredStore;
|
pub use layered_store::LayeredStore;
|
||||||
pub use map::LayeredMap;
|
pub use map::LayeredMap;
|
||||||
pub use meta::{IndexMode, PartitionMeta};
|
pub use meta::{IndexMode, PartitionMeta};
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ use obikseq::CanonicalKmer;
|
|||||||
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
|
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
|
||||||
|
|
||||||
use crate::error::{OLMError, OLMResult};
|
use crate::error::{OLMError, OLMResult};
|
||||||
use crate::layer::{layer_dir, Hit, Layer, LayerContent, LayerData};
|
use crate::layer::{layer_dir, Hit, TypedLayer, LayerContent, LayerData};
|
||||||
use crate::meta::{IndexMode, PartitionMeta};
|
use crate::meta::{IndexMode, PartitionMeta};
|
||||||
use crate::mphf_layer::EvidenceKind;
|
use crate::mphf_layer::EvidenceKind;
|
||||||
|
|
||||||
@@ -19,7 +19,7 @@ use crate::mphf_layer::EvidenceKind;
|
|||||||
pub struct LayeredMap<D: LayerData = ()> {
|
pub struct LayeredMap<D: LayerData = ()> {
|
||||||
root: PathBuf,
|
root: PathBuf,
|
||||||
meta: PartitionMeta,
|
meta: PartitionMeta,
|
||||||
layers: Vec<Layer<D>>,
|
layers: Vec<TypedLayer<D>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Common methods ────────────────────────────────────────────────────────────
|
// ── Common methods ────────────────────────────────────────────────────────────
|
||||||
@@ -30,7 +30,7 @@ impl<D: LayerData> LayeredMap<D> {
|
|||||||
pub fn open(root: &Path) -> OLMResult<Self> {
|
pub fn open(root: &Path) -> OLMResult<Self> {
|
||||||
let meta = PartitionMeta::load(root)?;
|
let meta = PartitionMeta::load(root)?;
|
||||||
let layers = (0..meta.n_layers)
|
let layers = (0..meta.n_layers)
|
||||||
.map(|i| Layer::<D>::open(&layer_dir(root, i), &meta.mode))
|
.map(|i| TypedLayer::<D>::open(&layer_dir(root, i), &meta.mode))
|
||||||
.collect::<OLMResult<Vec<_>>>()?;
|
.collect::<OLMResult<Vec<_>>>()?;
|
||||||
Ok(Self { root: root.to_owned(), meta, layers })
|
Ok(Self { root: root.to_owned(), meta, layers })
|
||||||
}
|
}
|
||||||
@@ -44,18 +44,18 @@ impl<D: LayerData> LayeredMap<D> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn n_layers(&self) -> usize { self.layers.len() }
|
pub fn n_layers(&self) -> usize { self.layers.len() }
|
||||||
pub fn layer(&self, i: usize) -> &Layer<D> { &self.layers[i] }
|
pub fn layer(&self, i: usize) -> &TypedLayer<D> { &self.layers[i] }
|
||||||
pub fn mode(&self) -> &IndexMode { &self.meta.mode }
|
pub fn mode(&self) -> &IndexMode { &self.meta.mode }
|
||||||
|
|
||||||
/// Layer `i`'s content (`Count`/`Presence`) — a lightweight disk probe,
|
/// TypedLayer `i`'s content (`Count`/`Presence`) — a lightweight disk probe,
|
||||||
/// no MPHF/matrix opened. For picking a `D` before committing to
|
/// no MPHF/matrix opened. For picking a `D` before committing to
|
||||||
/// [`Layer::open`], or for reporting/diagnostics over an already-open
|
/// [`TypedLayer::open`], or for reporting/diagnostics over an already-open
|
||||||
/// `LayeredMap` without re-opening every layer under a different `D`.
|
/// `LayeredMap` without re-opening every layer under a different `D`.
|
||||||
pub fn detect_layer_content(&self, i: usize) -> LayerContent {
|
pub fn detect_layer_content(&self, i: usize) -> LayerContent {
|
||||||
LayerContent::detect(&layer_dir(&self.root, i))
|
LayerContent::detect(&layer_dir(&self.root, i))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Layer `i`'s storage format — a lightweight disk probe, no
|
/// TypedLayer `i`'s storage format — a lightweight disk probe, no
|
||||||
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
|
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
|
||||||
pub fn detect_layer_storage(&self, i: usize) -> OLMResult<StorageKind> {
|
pub fn detect_layer_storage(&self, i: usize) -> OLMResult<StorageKind> {
|
||||||
let dir = layer_dir(&self.root, i);
|
let dir = layer_dir(&self.root, i);
|
||||||
@@ -65,7 +65,7 @@ impl<D: LayerData> LayeredMap<D> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Layer `i`'s evidence mode — a lightweight disk probe, no
|
/// TypedLayer `i`'s evidence mode — a lightweight disk probe, no
|
||||||
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
|
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
|
||||||
pub fn detect_layer_evidence(&self, i: usize) -> OLMResult<EvidenceKind> {
|
pub fn detect_layer_evidence(&self, i: usize) -> OLMResult<EvidenceKind> {
|
||||||
EvidenceKind::detect(&layer_dir(&self.root, i))
|
EvidenceKind::detect(&layer_dir(&self.root, i))
|
||||||
@@ -81,13 +81,13 @@ impl<D: LayerData> LayeredMap<D> {
|
|||||||
|
|
||||||
pub fn next_layer_writer(&self) -> OLMResult<UnitigFileWriter> {
|
pub fn next_layer_writer(&self) -> OLMResult<UnitigFileWriter> {
|
||||||
let dir = layer_dir(&self.root, self.layers.len());
|
let dir = layer_dir(&self.root, self.layers.len());
|
||||||
Layer::<D>::unitig_writer(&dir)
|
TypedLayer::<D>::unitig_writer(&dir)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn append_layer(&mut self) -> OLMResult<()> {
|
fn append_layer(&mut self) -> OLMResult<()> {
|
||||||
let i = self.layers.len();
|
let i = self.layers.len();
|
||||||
let dir = layer_dir(&self.root, i);
|
let dir = layer_dir(&self.root, i);
|
||||||
self.layers.push(Layer::<D>::open(&dir, &self.meta.mode)?);
|
self.layers.push(TypedLayer::<D>::open(&dir, &self.meta.mode)?);
|
||||||
self.meta.n_layers = self.layers.len();
|
self.meta.n_layers = self.layers.len();
|
||||||
self.meta.save(&self.root)?;
|
self.meta.save(&self.root)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -100,7 +100,7 @@ impl LayeredMap<()> {
|
|||||||
pub fn push_layer(&mut self) -> OLMResult<usize> {
|
pub fn push_layer(&mut self) -> OLMResult<usize> {
|
||||||
let i = self.layers.len();
|
let i = self.layers.len();
|
||||||
let dir = layer_dir(&self.root, i);
|
let dir = layer_dir(&self.root, i);
|
||||||
Layer::<()>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode)?;
|
TypedLayer::<()>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode)?;
|
||||||
self.append_layer()?;
|
self.append_layer()?;
|
||||||
Ok(i)
|
Ok(i)
|
||||||
}
|
}
|
||||||
@@ -112,7 +112,7 @@ impl LayeredMap<PersistentCompactIntMatrix> {
|
|||||||
pub fn push_layer(&mut self, count_of: impl Fn(CanonicalKmer) -> u32) -> OLMResult<usize> {
|
pub fn push_layer(&mut self, count_of: impl Fn(CanonicalKmer) -> u32) -> OLMResult<usize> {
|
||||||
let i = self.layers.len();
|
let i = self.layers.len();
|
||||||
let dir = layer_dir(&self.root, i);
|
let dir = layer_dir(&self.root, i);
|
||||||
Layer::<PersistentCompactIntMatrix>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode, count_of)?;
|
TypedLayer::<PersistentCompactIntMatrix>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode, count_of)?;
|
||||||
self.append_layer()?;
|
self.append_layer()?;
|
||||||
Ok(i)
|
Ok(i)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ fn canonical_kmer_iter_matches_reader() {
|
|||||||
assert_eq!(from_iter, from_reader, "CanonicalKmerIter and UnitigFileReader disagree");
|
assert_eq!(from_iter, from_reader, "CanonicalKmerIter and UnitigFileReader disagree");
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Generic `Layer<D>` over dense vs. sparse presence storage ───────────────
|
// ── Generic `TypedLayer<D>` over dense vs. sparse presence storage ───────────────
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn presence_layer_generic_over_sparse_matches_dense() {
|
fn presence_layer_generic_over_sparse_matches_dense() {
|
||||||
@@ -64,11 +64,11 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
|||||||
// kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be
|
// kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be
|
||||||
// biologically meaningful, just the same on both sides of the
|
// biologically meaningful, just the same on both sides of the
|
||||||
// dense/sparse comparison below.
|
// dense/sparse comparison below.
|
||||||
Layer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
|
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
|
||||||
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
||||||
}).unwrap();
|
}).unwrap();
|
||||||
|
|
||||||
let dense_layer = Layer::<PersistentBitMatrix>::open(dir.path(), &mode).unwrap();
|
let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path(), &mode).unwrap();
|
||||||
assert!(dense_layer.n_cols() >= 1);
|
assert!(dense_layer.n_cols() >= 1);
|
||||||
|
|
||||||
// Build the sparse form directly into the same `presence/` dir the
|
// Build the sparse form directly into the same `presence/` dir the
|
||||||
@@ -81,7 +81,7 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
|||||||
.close()
|
.close()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let sparse_layer = Layer::<PersistentSparseBitMatrix>::open(dir.path(), &mode).unwrap();
|
let sparse_layer = TypedLayer::<PersistentSparseBitMatrix>::open(dir.path(), &mode).unwrap();
|
||||||
|
|
||||||
// Same generic methods, same results, different concrete `D`.
|
// Same generic methods, same results, different concrete `D`.
|
||||||
assert_eq!(dense_layer.n_cols(), sparse_layer.n_cols());
|
assert_eq!(dense_layer.n_cols(), sparse_layer.n_cols());
|
||||||
@@ -91,7 +91,7 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
|||||||
assert_eq!(dense_layer.sub_matrix(&slots), sparse_layer.sub_matrix(&slots));
|
assert_eq!(dense_layer.sub_matrix(&slots), sparse_layer.sub_matrix(&slots));
|
||||||
|
|
||||||
// The matrix-agnostic, MPHF-only surface (from the generic
|
// The matrix-agnostic, MPHF-only surface (from the generic
|
||||||
// `impl<D: LayerData> Layer<D>`) must also agree: same kmer set,
|
// `impl<D: LayerData> TypedLayer<D>`) must also agree: same kmer set,
|
||||||
// looked up through either concrete `D`.
|
// looked up through either concrete `D`.
|
||||||
for kmer in all_canonical_kmers(dir.path()) {
|
for kmer in all_canonical_kmers(dir.path()) {
|
||||||
assert_eq!(dense_layer.find_slot(kmer), sparse_layer.find_slot(kmer));
|
assert_eq!(dense_layer.find_slot(kmer), sparse_layer.find_slot(kmer));
|
||||||
@@ -105,9 +105,9 @@ fn count_layer_reports_count_content_and_columnar_storage() {
|
|||||||
set_k(4);
|
set_k(4);
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
write_unitigs(dir.path(), &[b"AAAACGT"]);
|
write_unitigs(dir.path(), &[b"AAAACGT"]);
|
||||||
Layer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
|
TypedLayer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let layer = Layer::<PersistentCompactIntMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
|
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
|
||||||
|
|
||||||
assert_eq!(layer.content(), LayerContent::Count);
|
assert_eq!(layer.content(), LayerContent::Count);
|
||||||
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
||||||
@@ -119,10 +119,10 @@ fn presence_layer_reports_presence_content_and_columnar_storage() {
|
|||||||
set_k(4);
|
set_k(4);
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
|
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
|
||||||
Layer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
|
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
|
||||||
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
||||||
}).unwrap();
|
}).unwrap();
|
||||||
let layer = Layer::<PersistentBitMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
|
let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
|
||||||
|
|
||||||
assert_eq!(layer.content(), LayerContent::Presence);
|
assert_eq!(layer.content(), LayerContent::Presence);
|
||||||
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
||||||
|
|||||||
@@ -125,7 +125,7 @@ fn presence_partition(seqs: &[&[u8]], n_genomes: usize) -> (tempfile::TempDir, L
|
|||||||
let mode = IndexMode::Exact;
|
let mode = IndexMode::Exact;
|
||||||
let layer0 = layer_dir(dir.path(), 0);
|
let layer0 = layer_dir(dir.path(), 0);
|
||||||
write_unitigs_at(&layer0, seqs);
|
write_unitigs_at(&layer0, seqs);
|
||||||
Layer::<PersistentBitMatrix>::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
|
TypedLayer::<PersistentBitMatrix>::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
|
||||||
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
||||||
}).unwrap();
|
}).unwrap();
|
||||||
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
|
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
|
||||||
@@ -173,7 +173,7 @@ fn detect_layer_storage_implicit() {
|
|||||||
write_unitigs_at(&layer0, &[b"AAAACGT"]);
|
write_unitigs_at(&layer0, &[b"AAAACGT"]);
|
||||||
// Mode-1 build: MPHF + evidence + `layer_meta.json`, no matrix at all —
|
// Mode-1 build: MPHF + evidence + `layer_meta.json`, no matrix at all —
|
||||||
// must read back as `Presence`/`Implicit`, never a third "empty" state.
|
// must read back as `Presence`/`Implicit`, never a third "empty" state.
|
||||||
Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap();
|
TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap();
|
||||||
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
|
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
|
||||||
let map = LayeredMap::<()>::open(dir.path()).unwrap();
|
let map = LayeredMap::<()>::open(dir.path()).unwrap();
|
||||||
|
|
||||||
@@ -203,7 +203,7 @@ fn detect_layer_evidence_exact_vs_approx() {
|
|||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let layer0 = layer_dir(dir.path(), 0);
|
let layer0 = layer_dir(dir.path(), 0);
|
||||||
write_unitigs_at(&layer0, &[b"AAAACGT"]);
|
write_unitigs_at(&layer0, &[b"AAAACGT"]);
|
||||||
Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap();
|
TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap();
|
||||||
PartitionMeta { n_layers: 1, mode: IndexMode::Exact }.save(dir.path()).unwrap();
|
PartitionMeta { n_layers: 1, mode: IndexMode::Exact }.save(dir.path()).unwrap();
|
||||||
let map = LayeredMap::<()>::open(dir.path()).unwrap();
|
let map = LayeredMap::<()>::open(dir.path()).unwrap();
|
||||||
assert_eq!(map.detect_layer_evidence(0).unwrap(), EvidenceKind::Exact);
|
assert_eq!(map.detect_layer_evidence(0).unwrap(), EvidenceKind::Exact);
|
||||||
@@ -212,7 +212,7 @@ fn detect_layer_evidence_exact_vs_approx() {
|
|||||||
let dir2 = tempdir().unwrap();
|
let dir2 = tempdir().unwrap();
|
||||||
let layer0b = layer_dir(dir2.path(), 0);
|
let layer0b = layer_dir(dir2.path(), 0);
|
||||||
write_unitigs_at(&layer0b, &[b"AAAACGT"]);
|
write_unitigs_at(&layer0b, &[b"AAAACGT"]);
|
||||||
Layer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap();
|
TypedLayer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap();
|
||||||
PartitionMeta { n_layers: 1, mode: approx }.save(dir2.path()).unwrap();
|
PartitionMeta { n_layers: 1, mode: approx }.save(dir2.path()).unwrap();
|
||||||
let map2 = LayeredMap::<()>::open(dir2.path()).unwrap();
|
let map2 = LayeredMap::<()>::open(dir2.path()).unwrap();
|
||||||
assert_eq!(map2.detect_layer_evidence(0).unwrap(), EvidenceKind::Approx);
|
assert_eq!(map2.detect_layer_evidence(0).unwrap(), EvidenceKind::Approx);
|
||||||
|
|||||||
Reference in New Issue
Block a user