Start of a huge refactoring
This commit is contained in:
1 parent
1261aeba86
commit
5048f43eea
1040 files changed
+1050
-270
No files matched your search
@@ -1,56 +1,80 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use obikindex::{KmerIndex, layer::KmerLayer};
|
||||
use obikseq::CanonicalKmer;
|
||||
|
||||
use crate::meta_cache::MetaCache;
|
||||
|
||||
pub struct IndexCache<'a> {
|
||||
/// Ties this cache's lifetime to the `KmerIndex` it was opened from —
|
||||
/// borrow-checker-enforced, not just documentation: an `IndexCache`
|
||||
/// cannot outlive its index, so it can never be read against an index
|
||||
/// that has since been mutated or dropped. `index()` is a genuine
|
||||
/// accessor, but the field's real job is this lifetime bound, kept even
|
||||
/// on paths that never call `index()`.
|
||||
raw_index: &'a KmerIndex,
|
||||
meta: MetaCache,
|
||||
|
||||
n_partition: usize,
|
||||
n_layer: usize,
|
||||
layer_cache: Vec<KmerLayer>,
|
||||
/// Keyed by each partition's own index number, not renumbered —
|
||||
/// `partitions: Some(vec![5])` still reads back as partition `5`
|
||||
/// everywhere (`get_layer(5, ..)`, `find_in_partition(5, ..)`, …), not
|
||||
/// `0`. A `HashMap` rather than a flat `Vec` because the set of cached
|
||||
/// partitions doesn't have to be `0..n_partitions`, or even contiguous
|
||||
/// — see [`new`](Self::new).
|
||||
layer_cache: HashMap<usize, Vec<KmerLayer>>,
|
||||
}
|
||||
|
||||
impl<'a> IndexCache<'a> {
|
||||
/// Opens every layer of every partition once, up front, and keeps them
|
||||
/// alive for the cache's lifetime — `layer(p, l)` stored flat at
|
||||
/// `p * n_layer + l` (`n_layer` uniform across partitions, guaranteed by
|
||||
/// construction).
|
||||
/// Opens every layer of the given `partitions` (every partition of the
|
||||
/// index, if `None`) once, up front, and keeps them alive for the
|
||||
/// cache's lifetime — each partition's own layer count is read
|
||||
/// independently (`KmerPartition::n_layers`), so partitions don't need
|
||||
/// a uniform layer count across each other. That matters for a caller
|
||||
/// like `obikmerge`, which only ever needs one specific partition of a
|
||||
/// destination index still being merged into elsewhere — the other
|
||||
/// partitions can legitimately have a different layer count at that
|
||||
/// moment (built in parallel, mid-run), without that affecting a cache
|
||||
/// scoped to just the one partition being read.
|
||||
///
|
||||
/// Panics if the index isn't fully built: a cache can only be built on a
|
||||
/// functional index, so any failure here (missing/incomplete layer) is a
|
||||
/// caller error, not a case to design around with a `Result`.
|
||||
pub fn new(index: &'a KmerIndex) -> Self {
|
||||
let n_partition = index.n_partitions();
|
||||
let n_layer = index.n_layers_per_partition().unwrap_or_else(|e| {
|
||||
panic!("IndexCache::new: index is not fully built (n_layers_per_partition failed): {e}")
|
||||
});
|
||||
/// Panics if any requested partition fails to open (e.g. an
|
||||
/// out-of-range partition number, or a layer that fails to open): a
|
||||
/// cache can only be built on partitions that actually exist and are
|
||||
/// complete, so any failure here is a caller error, not a case to
|
||||
/// design around with a `Result`.
|
||||
pub fn new(index: &'a KmerIndex, partitions: Option<Vec<usize>>) -> Self {
|
||||
let selected = partitions.unwrap_or_else(|| (0..index.n_partitions()).collect());
|
||||
|
||||
let mut layer_cache = Vec::with_capacity(n_partition * n_layer);
|
||||
for p in 0..n_partition {
|
||||
let mut layer_cache = HashMap::with_capacity(selected.len());
|
||||
for p in selected {
|
||||
let partition = index
|
||||
.partition(p)
|
||||
.unwrap_or_else(|e| panic!("IndexCache::new: failed to open partition {p}: {e}"));
|
||||
let n_layer = partition.n_layers();
|
||||
let mut layers = Vec::with_capacity(n_layer);
|
||||
for l in 0..n_layer {
|
||||
let layer = index
|
||||
.layer(p, l)
|
||||
.and_then(KmerLayer::open)
|
||||
.unwrap_or_else(|e| {
|
||||
panic!(
|
||||
"IndexCache::new: failed to open layer (partition {p}, layer {l}): {e} — index is not fully built"
|
||||
)
|
||||
});
|
||||
layer_cache.push(layer);
|
||||
let layer = partition.layer(l).and_then(KmerLayer::open).unwrap_or_else(|e| {
|
||||
panic!(
|
||||
"IndexCache::new: failed to open layer (partition {p}, layer {l}): {e} — partition is not fully built"
|
||||
)
|
||||
});
|
||||
layers.push(layer);
|
||||
}
|
||||
layer_cache.insert(p, layers);
|
||||
}
|
||||
|
||||
IndexCache {
|
||||
raw_index: index,
|
||||
meta: MetaCache::from_meta(&index.meta()),
|
||||
n_partition,
|
||||
n_layer,
|
||||
layer_cache,
|
||||
}
|
||||
}
|
||||
|
||||
/// The index this cache was opened from.
|
||||
#[inline]
|
||||
pub fn index(&self) -> &'a KmerIndex {
|
||||
self.raw_index
|
||||
}
|
||||
|
||||
/// The index's metadata, snapshotted once at construction — see
|
||||
/// [`MetaCache`]. Read-only
|
||||
#[inline]
|
||||
@@ -58,31 +82,41 @@ impl<'a> IndexCache<'a> {
|
||||
&self.meta
|
||||
}
|
||||
|
||||
/// Number of layers cached for `partition` — `None` if `partition`
|
||||
/// isn't part of this cache.
|
||||
#[inline]
|
||||
pub fn n_layer(&self) -> usize {
|
||||
self.n_layer
|
||||
pub fn n_layer(&self, partition: usize) -> Option<usize> {
|
||||
Some(self.layer_cache.get(&partition)?.len())
|
||||
}
|
||||
|
||||
/// Number of partitions cached — not necessarily `raw_index.n_partitions()`
|
||||
/// when this cache was built on a subset (see [`new`](Self::new)).
|
||||
#[inline]
|
||||
pub fn n_partition(&self) -> usize {
|
||||
self.n_partition
|
||||
self.layer_cache.len()
|
||||
}
|
||||
|
||||
/// Which partitions this cache holds, in no particular order.
|
||||
pub fn partitions(&self) -> impl Iterator<Item = usize> + '_ {
|
||||
self.layer_cache.keys().copied()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get_layer(&self, partition: usize, layer: usize) -> Option<&KmerLayer> {
|
||||
self.layer_cache.get(partition * self.n_layer + layer)
|
||||
self.layer_cache.get(&partition)?.get(layer)
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = &KmerLayer> {
|
||||
self.layer_cache.iter()
|
||||
self.layer_cache.values().flatten()
|
||||
}
|
||||
|
||||
pub fn iter_indexed(&self) -> impl Iterator<Item = (usize, usize, &KmerLayer)> {
|
||||
let n_layer = self.n_layer;
|
||||
self.layer_cache
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(move |(i, layer)| (i / n_layer, i % n_layer, layer))
|
||||
self.layer_cache.iter().flat_map(|(&p, layers)| {
|
||||
layers
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(move |(l, layer)| (p, l, layer))
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -99,19 +133,25 @@ impl<'a> IndexCache<'a> {
|
||||
/// goes through `KmerLayer::find_slot`'s evidence check, not a blind
|
||||
/// MPHF lookup.
|
||||
#[inline]
|
||||
pub fn find_in_layer(&self, partition: usize, layer: usize, kmer: CanonicalKmer) -> Option<usize> {
|
||||
pub fn find_in_layer(
|
||||
&self,
|
||||
partition: usize,
|
||||
layer: usize,
|
||||
kmer: CanonicalKmer,
|
||||
) -> Option<usize> {
|
||||
self.get_layer(partition, layer)?.find_slot(kmer)
|
||||
}
|
||||
|
||||
/// Membership-checked lookup across every layer of one partition —
|
||||
/// first layer that carries `kmer` wins. `None` if `partition` is out
|
||||
/// of range or no layer of it carries `kmer`.
|
||||
/// Membership-checked lookup across every cached layer of one
|
||||
/// partition — first layer that carries `kmer` wins. `None` if
|
||||
/// `partition` isn't cached or no layer of it carries `kmer`.
|
||||
pub fn find_in_partition(
|
||||
&self,
|
||||
partition: usize,
|
||||
kmer: CanonicalKmer,
|
||||
) -> Option<(usize, usize, usize)> {
|
||||
for l in 0..self.n_layer {
|
||||
let n_layer = self.n_layer(partition)?;
|
||||
for l in 0..n_layer {
|
||||
if let Some(slot) = self.find_in_layer(partition, l, kmer) {
|
||||
return Some((partition, l, slot));
|
||||
}
|
||||
@@ -119,14 +159,17 @@ impl<'a> IndexCache<'a> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Membership-checked lookup across every partition and layer — first
|
||||
/// one that carries `kmer` wins.
|
||||
/// Membership-checked lookup — routes `kmer` straight to its one
|
||||
/// possible partition (`CanonicalKmer::partition`, the same
|
||||
/// minimizer-hash rule every k-mer was scattered by when the index was
|
||||
/// built) instead of scanning every cached partition: a k-mer can only
|
||||
/// ever live in the partition it hashes to, so anywhere else is not
|
||||
/// worth even asking. Uses `self.index().n_partitions()` — the index's
|
||||
/// real total, not [`n_partition`](Self::n_partition) (this cache's own
|
||||
/// count, which can be a subset — see [`new`](Self::new)). `None` if
|
||||
/// that partition isn't cached or doesn't carry `kmer`.
|
||||
pub fn find(&self, kmer: CanonicalKmer) -> Option<(usize, usize, usize)> {
|
||||
for p in 0..self.n_partition {
|
||||
if let Some(hit) = self.find_in_partition(p, kmer) {
|
||||
return Some(hit);
|
||||
}
|
||||
}
|
||||
None
|
||||
let partition = kmer.partition(self.raw_index.n_partitions());
|
||||
self.find_in_partition(partition, kmer)
|
||||
}
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
use ndarray::{Array1, Array2};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use obicompactvec::traits::{BitPartials, ColumnWeights, CountPartials};
|
||||
|
||||
/// A store that aggregates a `Vec<S>` — one entry per layer (within a partition)
|
||||
/// or one entry per partition.
|
||||
///
|
||||
/// Blanket impls of `ColumnWeights`, `CountPartials`, and `BitPartials` propagate
|
||||
/// automatically: `LayeredStore<LayeredStore<S>>` implements the same traits as
|
||||
/// `LayeredStore<S>`, giving the partitioned level for free.
|
||||
pub struct LayeredStore<S>(pub Vec<S>);
|
||||
|
||||
impl<S> LayeredStore<S> {
|
||||
pub fn new(layers: Vec<S>) -> Self { Self(layers) }
|
||||
pub fn layers(&self) -> &[S] { &self.0 }
|
||||
pub fn n_layers(&self) -> usize { self.0.len() }
|
||||
pub fn is_empty(&self) -> bool { self.0.is_empty() }
|
||||
}
|
||||
|
||||
// ── ColumnWeights ─────────────────────────────────────────────────────────────
|
||||
|
||||
impl<S: ColumnWeights> ColumnWeights for LayeredStore<S> {
|
||||
fn col_weights(&self) -> Array1<u64> {
|
||||
self.0.par_iter()
|
||||
.map(|s| s.col_weights())
|
||||
.reduce_with(|a, b| a + b)
|
||||
.unwrap_or_else(|| Array1::zeros(0))
|
||||
}
|
||||
}
|
||||
|
||||
// ── CountPartials ─────────────────────────────────────────────────────────────
|
||||
|
||||
impl<S: CountPartials> CountPartials for LayeredStore<S> {
|
||||
fn partial_bray(&self) -> Array2<u64> {
|
||||
self.0.par_iter()
|
||||
.map(|s| s.partial_bray())
|
||||
.reduce_with(|a, b| a + b)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn partial_euclidean(&self) -> Array2<f64> {
|
||||
self.0.par_iter()
|
||||
.map(|s| s.partial_euclidean())
|
||||
.reduce_with(|a, b| a + b)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn partial_threshold_jaccard(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
|
||||
self.0.par_iter()
|
||||
.map(|s| s.partial_threshold_jaccard(threshold))
|
||||
.reduce_with(|(ai, au), (bi, bu)| (ai + bi, au + bu))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn partial_relfreq_bray(&self, global: &Array1<u64>) -> Array2<f64> {
|
||||
self.0.par_iter()
|
||||
.map(|s| s.partial_relfreq_bray(global))
|
||||
.reduce_with(|a, b| a + b)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn partial_relfreq_euclidean(&self, global: &Array1<u64>) -> Array2<f64> {
|
||||
self.0.par_iter()
|
||||
.map(|s| s.partial_relfreq_euclidean(global))
|
||||
.reduce_with(|a, b| a + b)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn partial_hellinger(&self, global: &Array1<u64>) -> Array2<f64> {
|
||||
self.0.par_iter()
|
||||
.map(|s| s.partial_hellinger(global))
|
||||
.reduce_with(|a, b| a + b)
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
// ── BitPartials ───────────────────────────────────────────────────────────────
|
||||
|
||||
impl<S: BitPartials> BitPartials for LayeredStore<S> {
|
||||
fn partial_jaccard(&self) -> (Array2<u64>, Array2<u64>) {
|
||||
self.0.par_iter()
|
||||
.map(|s| s.partial_jaccard())
|
||||
.reduce_with(|(ai, au), (bi, bu)| (ai + bi, au + bu))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn partial_hamming(&self) -> Array2<u64> {
|
||||
self.0.par_iter()
|
||||
.map(|s| s.partial_hamming())
|
||||
.reduce_with(|a, b| a + b)
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tests/layered_store.rs"]
|
||||
mod tests;
|
||||
Reference in new issue
Block a user