Start of a huge refactoring

This commit is contained in:
Eric Coissac committed 2026-08-22 17:03:53 +02:00
1 parent 1261aeba86
commit 5048f43eea
1040 files changed
+1050 -270

No files matched your search

+94 -51
View File
@@ -1,56 +1,80 @@
use std::collections::HashMap;
use obikindex::{KmerIndex, layer::KmerLayer};
use obikseq::CanonicalKmer;
use crate::meta_cache::MetaCache;
pub struct IndexCache<'a> {
/// Ties this cache's lifetime to the `KmerIndex` it was opened from —
/// borrow-checker-enforced, not just documentation: an `IndexCache`
/// cannot outlive its index, so it can never be read against an index
/// that has since been mutated or dropped. `index()` is a genuine
/// accessor, but the field's real job is this lifetime bound, kept even
/// on paths that never call `index()`.
raw_index: &'a KmerIndex,
meta: MetaCache,
n_partition: usize,
n_layer: usize,
layer_cache: Vec<KmerLayer>,
/// Keyed by each partition's own index number, not renumbered —
/// `partitions: Some(vec![5])` still reads back as partition `5`
/// everywhere (`get_layer(5, ..)`, `find_in_partition(5, ..)`, …), not
/// `0`. A `HashMap` rather than a flat `Vec` because the set of cached
/// partitions doesn't have to be `0..n_partitions`, or even contiguous
/// — see [`new`](Self::new).
layer_cache: HashMap<usize, Vec<KmerLayer>>,
}
impl<'a> IndexCache<'a> {
/// Opens every layer of every partition once, up front, and keeps them
/// alive for the cache's lifetime — `layer(p, l)` stored flat at
/// `p * n_layer + l` (`n_layer` uniform across partitions, guaranteed by
/// construction).
/// Opens every layer of the given `partitions` (every partition of the
/// index, if `None`) once, up front, and keeps them alive for the
/// cache's lifetime — each partition's own layer count is read
/// independently (`KmerPartition::n_layers`), so partitions don't need
/// a uniform layer count across each other. That matters for a caller
/// like `obikmerge`, which only ever needs one specific partition of a
/// destination index still being merged into elsewhere — the other
/// partitions can legitimately have a different layer count at that
/// moment (built in parallel, mid-run), without that affecting a cache
/// scoped to just the one partition being read.
///
/// Panics if the index isn't fully built: a cache can only be built on a
/// functional index, so any failure here (missing/incomplete layer) is a
/// caller error, not a case to design around with a `Result`.
pub fn new(index: &'a KmerIndex) -> Self {
let n_partition = index.n_partitions();
let n_layer = index.n_layers_per_partition().unwrap_or_else(|e| {
panic!("IndexCache::new: index is not fully built (n_layers_per_partition failed): {e}")
});
/// Panics if any requested partition fails to open (e.g. an
/// out-of-range partition number, or a layer that fails to open): a
/// cache can only be built on partitions that actually exist and are
/// complete, so any failure here is a caller error, not a case to
/// design around with a `Result`.
pub fn new(index: &'a KmerIndex, partitions: Option<Vec<usize>>) -> Self {
let selected = partitions.unwrap_or_else(|| (0..index.n_partitions()).collect());
let mut layer_cache = Vec::with_capacity(n_partition * n_layer);
for p in 0..n_partition {
let mut layer_cache = HashMap::with_capacity(selected.len());
for p in selected {
let partition = index
.partition(p)
.unwrap_or_else(|e| panic!("IndexCache::new: failed to open partition {p}: {e}"));
let n_layer = partition.n_layers();
let mut layers = Vec::with_capacity(n_layer);
for l in 0..n_layer {
let layer = index
.layer(p, l)
.and_then(KmerLayer::open)
.unwrap_or_else(|e| {
panic!(
"IndexCache::new: failed to open layer (partition {p}, layer {l}): {e} — index is not fully built"
)
});
layer_cache.push(layer);
let layer = partition.layer(l).and_then(KmerLayer::open).unwrap_or_else(|e| {
panic!(
"IndexCache::new: failed to open layer (partition {p}, layer {l}): {e} — partition is not fully built"
)
});
layers.push(layer);
}
layer_cache.insert(p, layers);
}
IndexCache {
raw_index: index,
meta: MetaCache::from_meta(&index.meta()),
n_partition,
n_layer,
layer_cache,
}
}
/// The index this cache was opened from.
#[inline]
pub fn index(&self) -> &'a KmerIndex {
self.raw_index
}
/// The index's metadata, snapshotted once at construction — see
/// [`MetaCache`]. Read-only
#[inline]
@@ -58,31 +82,41 @@ impl<'a> IndexCache<'a> {
&self.meta
}
/// Number of layers cached for `partition` — `None` if `partition`
/// isn't part of this cache.
#[inline]
pub fn n_layer(&self) -> usize {
self.n_layer
pub fn n_layer(&self, partition: usize) -> Option<usize> {
Some(self.layer_cache.get(&partition)?.len())
}
/// Number of partitions cached — not necessarily `raw_index.n_partitions()`
/// when this cache was built on a subset (see [`new`](Self::new)).
#[inline]
pub fn n_partition(&self) -> usize {
self.n_partition
self.layer_cache.len()
}
/// Which partitions this cache holds, in no particular order.
pub fn partitions(&self) -> impl Iterator<Item = usize> + '_ {
self.layer_cache.keys().copied()
}
#[inline]
pub fn get_layer(&self, partition: usize, layer: usize) -> Option<&KmerLayer> {
self.layer_cache.get(partition * self.n_layer + layer)
self.layer_cache.get(&partition)?.get(layer)
}
pub fn iter(&self) -> impl Iterator<Item = &KmerLayer> {
self.layer_cache.iter()
self.layer_cache.values().flatten()
}
pub fn iter_indexed(&self) -> impl Iterator<Item = (usize, usize, &KmerLayer)> {
let n_layer = self.n_layer;
self.layer_cache
.iter()
.enumerate()
.map(move |(i, layer)| (i / n_layer, i % n_layer, layer))
self.layer_cache.iter().flat_map(|(&p, layers)| {
layers
.iter()
.enumerate()
.map(move |(l, layer)| (p, l, layer))
})
}
#[inline]
@@ -99,19 +133,25 @@ impl<'a> IndexCache<'a> {
/// goes through `KmerLayer::find_slot`'s evidence check, not a blind
/// MPHF lookup.
#[inline]
pub fn find_in_layer(&self, partition: usize, layer: usize, kmer: CanonicalKmer) -> Option<usize> {
pub fn find_in_layer(
&self,
partition: usize,
layer: usize,
kmer: CanonicalKmer,
) -> Option<usize> {
self.get_layer(partition, layer)?.find_slot(kmer)
}
/// Membership-checked lookup across every layer of one partition —
/// first layer that carries `kmer` wins. `None` if `partition` is out
/// of range or no layer of it carries `kmer`.
/// Membership-checked lookup across every cached layer of one
/// partition — first layer that carries `kmer` wins. `None` if
/// `partition` isn't cached or no layer of it carries `kmer`.
pub fn find_in_partition(
&self,
partition: usize,
kmer: CanonicalKmer,
) -> Option<(usize, usize, usize)> {
for l in 0..self.n_layer {
let n_layer = self.n_layer(partition)?;
for l in 0..n_layer {
if let Some(slot) = self.find_in_layer(partition, l, kmer) {
return Some((partition, l, slot));
}
@@ -119,14 +159,17 @@ impl<'a> IndexCache<'a> {
None
}
/// Membership-checked lookup across every partition and layer — first
/// one that carries `kmer` wins.
/// Membership-checked lookup — routes `kmer` straight to its one
/// possible partition (`CanonicalKmer::partition`, the same
/// minimizer-hash rule every k-mer was scattered by when the index was
/// built) instead of scanning every cached partition: a k-mer can only
/// ever live in the partition it hashes to, so anywhere else is not
/// worth even asking. Uses `self.index().n_partitions()` — the index's
/// real total, not [`n_partition`](Self::n_partition) (this cache's own
/// count, which can be a subset — see [`new`](Self::new)). `None` if
/// that partition isn't cached or doesn't carry `kmer`.
pub fn find(&self, kmer: CanonicalKmer) -> Option<(usize, usize, usize)> {
for p in 0..self.n_partition {
if let Some(hit) = self.find_in_partition(p, kmer) {
return Some(hit);
}
}
None
let partition = kmer.partition(self.raw_index.n_partitions());
self.find_in_partition(partition, kmer)
}
}
-100
View File
@@ -1,100 +0,0 @@
use ndarray::{Array1, Array2};
use rayon::prelude::*;
use obicompactvec::traits::{BitPartials, ColumnWeights, CountPartials};
/// A store that aggregates a `Vec<S>` — one entry per layer (within a partition)
/// or one entry per partition.
///
/// Blanket impls of `ColumnWeights`, `CountPartials`, and `BitPartials` propagate
/// automatically: `LayeredStore<LayeredStore<S>>` implements the same traits as
/// `LayeredStore<S>`, giving the partitioned level for free.
pub struct LayeredStore<S>(pub Vec<S>);
impl<S> LayeredStore<S> {
pub fn new(layers: Vec<S>) -> Self { Self(layers) }
pub fn layers(&self) -> &[S] { &self.0 }
pub fn n_layers(&self) -> usize { self.0.len() }
pub fn is_empty(&self) -> bool { self.0.is_empty() }
}
// ── ColumnWeights ─────────────────────────────────────────────────────────────
impl<S: ColumnWeights> ColumnWeights for LayeredStore<S> {
fn col_weights(&self) -> Array1<u64> {
self.0.par_iter()
.map(|s| s.col_weights())
.reduce_with(|a, b| a + b)
.unwrap_or_else(|| Array1::zeros(0))
}
}
// ── CountPartials ─────────────────────────────────────────────────────────────
impl<S: CountPartials> CountPartials for LayeredStore<S> {
fn partial_bray(&self) -> Array2<u64> {
self.0.par_iter()
.map(|s| s.partial_bray())
.reduce_with(|a, b| a + b)
.unwrap()
}
fn partial_euclidean(&self) -> Array2<f64> {
self.0.par_iter()
.map(|s| s.partial_euclidean())
.reduce_with(|a, b| a + b)
.unwrap()
}
fn partial_threshold_jaccard(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
self.0.par_iter()
.map(|s| s.partial_threshold_jaccard(threshold))
.reduce_with(|(ai, au), (bi, bu)| (ai + bi, au + bu))
.unwrap()
}
fn partial_relfreq_bray(&self, global: &Array1<u64>) -> Array2<f64> {
self.0.par_iter()
.map(|s| s.partial_relfreq_bray(global))
.reduce_with(|a, b| a + b)
.unwrap()
}
fn partial_relfreq_euclidean(&self, global: &Array1<u64>) -> Array2<f64> {
self.0.par_iter()
.map(|s| s.partial_relfreq_euclidean(global))
.reduce_with(|a, b| a + b)
.unwrap()
}
fn partial_hellinger(&self, global: &Array1<u64>) -> Array2<f64> {
self.0.par_iter()
.map(|s| s.partial_hellinger(global))
.reduce_with(|a, b| a + b)
.unwrap()
}
}
// ── BitPartials ───────────────────────────────────────────────────────────────
impl<S: BitPartials> BitPartials for LayeredStore<S> {
fn partial_jaccard(&self) -> (Array2<u64>, Array2<u64>) {
self.0.par_iter()
.map(|s| s.partial_jaccard())
.reduce_with(|(ai, au), (bi, bu)| (ai + bi, au + bu))
.unwrap()
}
fn partial_hamming(&self) -> Array2<u64> {
self.0.par_iter()
.map(|s| s.partial_hamming())
.reduce_with(|a, b| a + b)
.unwrap()
}
}
// ── Tests ─────────────────────────────────────────────────────────────────────
#[cfg(test)]
#[path = "tests/layered_store.rs"]
mod tests;