Introduce concurrency-bounded parallel processing for index cache layers
Implement an `obipipeline::Throttle` with an RAII guard to acquire and release concurrency slots. Expose new bounded parallel methods on `IndexCache` to process cached layers with a configurable cap. Refactor downstream aggregation logic to use single-pass parallel map-reduce instead of manual collect-map-reduce sequences, enforcing a maximum of 8 concurrent layer scans to bound memory usage.
This commit is contained in:
1 parent
9dee6dcd08
commit
d084396aba
6 files changed
+211
-62
No files matched your search
@@ -7,6 +7,7 @@ edition = "2024"
|
||||
obikindex = { path = "../obikindex"}
|
||||
obikseq = { path = "../obikseq"}
|
||||
obicompactvec = { path = "../obicompactvec" }
|
||||
obipipeline = { path = "../obipipeline" }
|
||||
ndarray = "0.17"
|
||||
rayon = "1"
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ use std::sync::Arc;
|
||||
|
||||
use obikindex::{KmerIndex, layer::KmerLayer};
|
||||
use obikseq::CanonicalKmer;
|
||||
use obipipeline::Throttle;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::meta_cache::MetaCache;
|
||||
|
||||
@@ -188,4 +190,86 @@ impl IndexCache {
|
||||
let partition = kmer.partition(self.raw_index.n_partitions());
|
||||
self.find_in_partition(partition, kmer)
|
||||
}
|
||||
|
||||
/// Parallel map-reduce over every cached `KmerLayer` (across every
|
||||
/// cached partition), bounded to `max_concurrent` layers scanned at
|
||||
/// once — a cap on how many layers are actively being mapped
|
||||
/// simultaneously, independent of `rayon`'s own thread count (a large
|
||||
/// per-layer working set, e.g. a whole matrix scan, can make "one
|
||||
/// layer per thread" too much resident memory at once even when the
|
||||
/// thread pool itself is much bigger).
|
||||
///
|
||||
/// Safe only when `map` is independent per layer (no cross-layer or
|
||||
/// cross-partition state) — when a caller's own per-layer work already
|
||||
/// parallelizes internally at a finer grain (e.g. across partitions,
|
||||
/// like `obikphylo`'s sibling-annex traversal), stacking this on top
|
||||
/// fights that inner parallelism for cache/mmap locality instead of
|
||||
/// helping; use the plain sequential [`iter`](Self::iter) there
|
||||
/// instead.
|
||||
pub fn par_map_reduce<T, M, R>(
|
||||
&self,
|
||||
max_concurrent: usize,
|
||||
identity: impl Fn() -> T + Sync,
|
||||
map: M,
|
||||
reduce: R,
|
||||
) -> T
|
||||
where
|
||||
T: Send,
|
||||
M: Fn(&KmerLayer) -> T + Sync,
|
||||
R: Fn(T, T) -> T + Sync,
|
||||
{
|
||||
let throttle = Arc::new(Throttle::new(max_concurrent));
|
||||
self.layer_cache
|
||||
.par_iter()
|
||||
.flat_map(|(_, layers)| layers.par_iter())
|
||||
.map(|layer| {
|
||||
let _guard = throttle.acquire_guard();
|
||||
map(layer)
|
||||
})
|
||||
.reduce(&identity, &reduce)
|
||||
}
|
||||
|
||||
/// Parallel, side-effecting foreach over every cached `KmerLayer`,
|
||||
/// bounded to `max_concurrent` layers at once — same bounding
|
||||
/// reasoning and same "independent per layer" safety requirement as
|
||||
/// [`par_map_reduce`](Self::par_map_reduce), just without a value to
|
||||
/// combine back.
|
||||
pub fn par_for_each<F>(&self, max_concurrent: usize, f: F)
|
||||
where
|
||||
F: Fn(&KmerLayer) + Sync,
|
||||
{
|
||||
let throttle = Arc::new(Throttle::new(max_concurrent));
|
||||
self.layer_cache
|
||||
.par_iter()
|
||||
.flat_map(|(_, layers)| layers.par_iter())
|
||||
.for_each(|layer| {
|
||||
let _guard = throttle.acquire_guard();
|
||||
f(layer);
|
||||
});
|
||||
}
|
||||
|
||||
/// Bounded parallel iterator over every cached `KmerLayer` — for
|
||||
/// callers that want to chain their own `rayon` combinators rather
|
||||
/// than fit their computation into [`par_map_reduce`](Self::par_map_reduce)/
|
||||
/// [`par_for_each`](Self::par_for_each)'s shape.
|
||||
///
|
||||
/// Each item is paired with a [`obipipeline::ThrottleGuard`] (same
|
||||
/// `Throttled<T>` carrier `obipipeline::throttle` uses for a plain
|
||||
/// sequential source) that releases its slot on drop — keep the whole
|
||||
/// `Throttled` alive for as long as you're "using" that layer (e.g.
|
||||
/// through your own `.map()`/`.for_each()`), the same discipline
|
||||
/// `obipipeline`'s own consumers already follow.
|
||||
pub fn par_iter_bounded(
|
||||
&self,
|
||||
max_concurrent: usize,
|
||||
) -> impl ParallelIterator<Item = obipipeline::Throttled<&KmerLayer>> {
|
||||
let throttle = Arc::new(Throttle::new(max_concurrent));
|
||||
self.layer_cache
|
||||
.par_iter()
|
||||
.flat_map(|(_, layers)| layers.par_iter())
|
||||
.map(move |layer| obipipeline::Throttled {
|
||||
item: layer,
|
||||
guard: throttle.acquire_guard(),
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user