Add per-family Shannon entropy calculation and reporting

Introduces an entropy.pent persisted format and algorithms to compute per-family Shannon entropy across genomic partitions. Refactors IndexCache to use Arc-based layer sharing for efficient iteration and restructures the siblings module with extension traits for annex generation. Adds a --shannon CLI flag to export partition-level entropy metrics to CSV.
This commit is contained in:
Eric Coissac committed 2026-08-28 20:45:47 +02:00
1 parent bbb58a698f
commit 5a0b71d105
14 files changed
+762 -136

No files matched your search

+104 -18
View File
@@ -28,7 +28,28 @@ pub struct IndexCache {
/// `0`. A `HashMap` rather than a flat `Vec` because the set of cached
/// partitions doesn't have to be `0..n_partitions`, or even contiguous
/// — see [`new`](Self::new).
layer_cache: HashMap<usize, Vec<KmerLayer>>,
///
/// `Arc<KmerLayer>` per layer, and the whole map itself behind one more
/// `Arc` — every accessor below hands out owned `Arc<KmerLayer>`
/// clones (never a `&KmerLayer` tied to `&self`'s lifetime), because
/// that's the only thing that can satisfy the `Send + 'static` bound
/// `obipipeline::Pipe::apply` (real `thread::spawn` workers) requires
/// of anything moved into them. A caller needing to hand one specific
/// cached layer to such a pipeline would otherwise have no way to do so
/// short of reopening it independently — a redundant mmap/MPHF open of
/// a layer this cache already holds (confirmed by
/// `obikphylo::siblings::algorithms::family_scan` hitting exactly this
/// before `Arc<KmerLayer>` was introduced here).
///
/// The outer `Arc<HashMap<..>>` is what lets [`iter`](Self::iter)/
/// [`iter_partition`](Self::iter_partition)/[`iter_indexed`](Self::iter_indexed)
/// return a genuine `Iterator` object (`LayerIter`, below) instead of
/// eagerly materialising every layer's `Arc` into a throwaway `Vec`
/// first: cloning the outer `Arc` is one refcount bump regardless of
/// how many layers this cache holds, so `LayerIter::next()` can look
/// up and clone one `Arc<KmerLayer>` at a time, on demand, straight out
/// of this same shared map — no separate payload copy ever exists.
layer_cache: Arc<HashMap<usize, Vec<Arc<KmerLayer>>>>,
}
impl IndexCache {
@@ -64,7 +85,7 @@ impl IndexCache {
"IndexCache::new: failed to open layer (partition {p}, layer {l}): {e} — partition is not fully built"
)
});
layers.push(layer);
layers.push(Arc::new(layer));
}
layer_cache.insert(p, layers);
}
@@ -73,7 +94,7 @@ impl IndexCache {
IndexCache {
raw_index: index,
meta,
layer_cache,
layer_cache: Arc::new(layer_cache),
}
}
@@ -109,22 +130,52 @@ impl IndexCache {
self.layer_cache.keys().copied()
}
/// A cheap `Arc` clone (refcount bump, no reopening) of one cached
/// layer — owned, not tied to `&self`'s lifetime, so it can be moved
/// into a context that must own it, e.g. a `Send + 'static` pipeline
/// worker. See the [`layer_cache`](Self::layer_cache) field docs for
/// why every accessor here hands out `Arc<KmerLayer>` rather than
/// `&KmerLayer`.
#[inline]
pub fn get_layer(&self, partition: usize, layer: usize) -> Option<&KmerLayer> {
self.layer_cache.get(&partition)?.get(layer)
pub fn get_layer(&self, partition: usize, layer: usize) -> Option<Arc<KmerLayer>> {
self.layer_cache.get(&partition)?.get(layer).cloned()
}
pub fn iter(&self) -> impl Iterator<Item = &KmerLayer> {
self.layer_cache.values().flatten()
/// Every cached layer, across every cached partition, as owned `Arc`
/// clones, in `(partition, layer)` order — genuinely `'static`: see
/// [`iter_indexed`](Self::iter_indexed)/[`LayerIter`]'s own docs.
pub fn iter(&self) -> impl Iterator<Item = Arc<KmerLayer>> + 'static {
self.iter_indexed().map(|(_, _, layer)| layer)
}
pub fn iter_indexed(&self) -> impl Iterator<Item = (usize, usize, &KmerLayer)> {
self.layer_cache.iter().flat_map(|(&p, layers)| {
layers
.iter()
.enumerate()
.map(move |(l, layer)| (p, l, layer))
})
/// Every cached layer of *one* partition, in layer order — never the
/// other cached partitions. For a caller that already knows which
/// partition a k-mer routes to (`CanonicalKmer::partition`, the same
/// rule [`find`](Self::find) uses) and only needs to scan that one
/// partition's own layers, not every partition this cache holds. Empty
/// (not an error) if `partition` isn't cached — same convention as
/// [`get_layer`](Self::get_layer). `'static` for the same reason
/// [`iter_indexed`](Self::iter_indexed) is.
pub fn iter_partition(&self, partition: usize) -> impl Iterator<Item = Arc<KmerLayer>> + 'static {
LayerIter {
cache: Arc::clone(&self.layer_cache),
partitions: vec![partition].into_iter(),
current: None,
}
.map(|(_, _, layer)| layer)
}
/// Every cached layer, tagged with its own `(partition, layer)`
/// position — see [`LayerIter`]'s own docs for why this is a real
/// `next()`-driven iterator, one `Arc<KmerLayer>` clone at a time out
/// of the shared cache map, rather than a `Vec<Arc<KmerLayer>>`
/// materialised up front.
pub fn iter_indexed(&self) -> impl Iterator<Item = (usize, usize, Arc<KmerLayer>)> + 'static {
LayerIter {
cache: Arc::clone(&self.layer_cache),
partitions: self.layer_cache.keys().copied().collect::<Vec<_>>().into_iter(),
current: None,
}
}
/// Iterate over the unitigs of every cached layer, chained — the
@@ -133,7 +184,7 @@ impl IndexCache {
/// (one partition, several, or the whole index — see [`new`](Self::new)).
/// No extra opening: every layer here is already open, this only
/// streams what [`iter`](Self::iter) already gives out.
pub fn iter_unitigs(&self) -> impl Iterator<Item = obikseq::Unitig> + '_ {
pub fn iter_unitigs(&self) -> impl Iterator<Item = obikseq::Unitig> + 'static {
self.iter().flat_map(|l| l.iter_unitigs())
}
@@ -224,7 +275,7 @@ impl IndexCache {
.flat_map(|(_, layers)| layers.par_iter())
.map(|layer| {
let _guard = throttle.acquire_guard();
map(layer)
map(layer.as_ref())
})
.reduce(&identity, &reduce)
}
@@ -244,7 +295,7 @@ impl IndexCache {
.flat_map(|(_, layers)| layers.par_iter())
.for_each(|layer| {
let _guard = throttle.acquire_guard();
f(layer);
f(layer.as_ref());
});
}
@@ -268,8 +319,43 @@ impl IndexCache {
.par_iter()
.flat_map(|(_, layers)| layers.par_iter())
.map(move |layer| obipipeline::Throttled {
item: layer,
item: layer.as_ref(),
guard: throttle.acquire_guard(),
})
}
}
/// A real `Iterator` (implements `next()`), not a `Vec` pre-filled and then
/// walked — `next()` looks up and clones exactly one `Arc<KmerLayer>` at a
/// time, straight out of `cache` (a clone of `IndexCache`'s own shared map,
/// one refcount bump to construct regardless of how many layers are cached),
/// so no separate copy of the layers being walked ever exists. `partitions`
/// is a small, already-owned `Vec<usize>` of which partition(s) to walk, in
/// order — holding a `Vec` of plain partition numbers costs nothing close
/// to what holding a `Vec` of the cached layers themselves would.
pub struct LayerIter {
cache: Arc<HashMap<usize, Vec<Arc<KmerLayer>>>>,
partitions: std::vec::IntoIter<usize>,
/// `(partition, next layer index)` of the partition currently being
/// walked — `None` once that partition is exhausted or before the
/// first `next()` call, at which point [`partitions`](Self::partitions)
/// is advanced to pick up the next one.
current: Option<(usize, usize)>,
}
impl Iterator for LayerIter {
type Item = (usize, usize, Arc<KmerLayer>);
fn next(&mut self) -> Option<Self::Item> {
loop {
if let Some((partition, layer_idx)) = self.current {
if let Some(layer) = self.cache.get(&partition).and_then(|v| v.get(layer_idx)) {
self.current = Some((partition, layer_idx + 1));
return Some((partition, layer_idx, Arc::clone(layer)));
}
self.current = None; // this partition exhausted
}
self.current = Some((self.partitions.next()?, 0));
}
}
}