Introduce concurrency-bounded parallel processing for index cache layers

Implement an `obipipeline::Throttle` with an RAII guard to acquire and release concurrency slots. Expose new bounded parallel methods on `IndexCache` to process cached layers with a configurable cap. Refactor downstream aggregation logic to use single-pass parallel map-reduce instead of manual collect-map-reduce sequences, enforcing a maximum of 8 concurrent layer scans to bound memory usage.
This commit is contained in:
Eric Coissac committed 2026-08-28 20:20:12 +02:00
1 parent 9dee6dcd08
commit d084396aba
6 files changed
+211 -62

No files matched your search

+1
View File
@@ -7,6 +7,7 @@ edition = "2024"
obikindex = { path = "../obikindex"}
obikseq = { path = "../obikseq"}
obicompactvec = { path = "../obicompactvec" }
obipipeline = { path = "../obipipeline" }
ndarray = "0.17"
rayon = "1"
+84
View File
@@ -3,6 +3,8 @@ use std::sync::Arc;
use obikindex::{KmerIndex, layer::KmerLayer};
use obikseq::CanonicalKmer;
use obipipeline::Throttle;
use rayon::prelude::*;
use crate::meta_cache::MetaCache;
@@ -188,4 +190,86 @@ impl IndexCache {
let partition = kmer.partition(self.raw_index.n_partitions());
self.find_in_partition(partition, kmer)
}
/// Parallel map-reduce over every cached `KmerLayer` (across every
/// cached partition), bounded to `max_concurrent` layers scanned at
/// once — a cap on how many layers are actively being mapped
/// simultaneously, independent of `rayon`'s own thread count (a large
/// per-layer working set, e.g. a whole matrix scan, can make "one
/// layer per thread" too much resident memory at once even when the
/// thread pool itself is much bigger).
///
/// Safe only when `map` is independent per layer (no cross-layer or
/// cross-partition state) — when a caller's own per-layer work already
/// parallelizes internally at a finer grain (e.g. across partitions,
/// like `obikphylo`'s sibling-annex traversal), stacking this on top
/// fights that inner parallelism for cache/mmap locality instead of
/// helping; use the plain sequential [`iter`](Self::iter) there
/// instead.
pub fn par_map_reduce<T, M, R>(
&self,
max_concurrent: usize,
identity: impl Fn() -> T + Sync,
map: M,
reduce: R,
) -> T
where
T: Send,
M: Fn(&KmerLayer) -> T + Sync,
R: Fn(T, T) -> T + Sync,
{
let throttle = Arc::new(Throttle::new(max_concurrent));
self.layer_cache
.par_iter()
.flat_map(|(_, layers)| layers.par_iter())
.map(|layer| {
let _guard = throttle.acquire_guard();
map(layer)
})
.reduce(&identity, &reduce)
}
/// Parallel, side-effecting foreach over every cached `KmerLayer`,
/// bounded to `max_concurrent` layers at once — same bounding
/// reasoning and same "independent per layer" safety requirement as
/// [`par_map_reduce`](Self::par_map_reduce), just without a value to
/// combine back.
pub fn par_for_each<F>(&self, max_concurrent: usize, f: F)
where
F: Fn(&KmerLayer) + Sync,
{
let throttle = Arc::new(Throttle::new(max_concurrent));
self.layer_cache
.par_iter()
.flat_map(|(_, layers)| layers.par_iter())
.for_each(|layer| {
let _guard = throttle.acquire_guard();
f(layer);
});
}
/// Bounded parallel iterator over every cached `KmerLayer` — for
/// callers that want to chain their own `rayon` combinators rather
/// than fit their computation into [`par_map_reduce`](Self::par_map_reduce)/
/// [`par_for_each`](Self::par_for_each)'s shape.
///
/// Each item is paired with a [`obipipeline::ThrottleGuard`] (same
/// `Throttled<T>` carrier `obipipeline::throttle` uses for a plain
/// sequential source) that releases its slot on drop — keep the whole
/// `Throttled` alive for as long as you're "using" that layer (e.g.
/// through your own `.map()`/`.for_each()`), the same discipline
/// `obipipeline`'s own consumers already follow.
pub fn par_iter_bounded(
&self,
max_concurrent: usize,
) -> impl ParallelIterator<Item = obipipeline::Throttled<&KmerLayer>> {
let throttle = Arc::new(Throttle::new(max_concurrent));
self.layer_cache
.par_iter()
.flat_map(|(_, layers)| layers.par_iter())
.map(move |layer| obipipeline::Throttled {
item: layer,
guard: throttle.acquire_guard(),
})
}
}