Add per-family Shannon entropy calculation and reporting
Introduces an entropy.pent persisted format and algorithms to compute per-family Shannon entropy across genomic partitions. Refactors IndexCache to use Arc-based layer sharing for efficient iteration and restructures the siblings module with extension traits for annex generation. Adds a --shannon CLI flag to export partition-level entropy metrics to CSV.
This commit is contained in:
1 parent
bbb58a698f
commit
5a0b71d105
14 files changed
+762
-136
No files matched your search
@@ -28,7 +28,28 @@ pub struct IndexCache {
|
||||
/// `0`. A `HashMap` rather than a flat `Vec` because the set of cached
|
||||
/// partitions doesn't have to be `0..n_partitions`, or even contiguous
|
||||
/// — see [`new`](Self::new).
|
||||
layer_cache: HashMap<usize, Vec<KmerLayer>>,
|
||||
///
|
||||
/// `Arc<KmerLayer>` per layer, and the whole map itself behind one more
|
||||
/// `Arc` — every accessor below hands out owned `Arc<KmerLayer>`
|
||||
/// clones (never a `&KmerLayer` tied to `&self`'s lifetime), because
|
||||
/// that's the only thing that can satisfy the `Send + 'static` bound
|
||||
/// `obipipeline::Pipe::apply` (real `thread::spawn` workers) requires
|
||||
/// of anything moved into them. A caller needing to hand one specific
|
||||
/// cached layer to such a pipeline would otherwise have no way to do so
|
||||
/// short of reopening it independently — a redundant mmap/MPHF open of
|
||||
/// a layer this cache already holds (confirmed by
|
||||
/// `obikphylo::siblings::algorithms::family_scan` hitting exactly this
|
||||
/// before `Arc<KmerLayer>` was introduced here).
|
||||
///
|
||||
/// The outer `Arc<HashMap<..>>` is what lets [`iter`](Self::iter)/
|
||||
/// [`iter_partition`](Self::iter_partition)/[`iter_indexed`](Self::iter_indexed)
|
||||
/// return a genuine `Iterator` object (`LayerIter`, below) instead of
|
||||
/// eagerly materialising every layer's `Arc` into a throwaway `Vec`
|
||||
/// first: cloning the outer `Arc` is one refcount bump regardless of
|
||||
/// how many layers this cache holds, so `LayerIter::next()` can look
|
||||
/// up and clone one `Arc<KmerLayer>` at a time, on demand, straight out
|
||||
/// of this same shared map — no separate payload copy ever exists.
|
||||
layer_cache: Arc<HashMap<usize, Vec<Arc<KmerLayer>>>>,
|
||||
}
|
||||
|
||||
impl IndexCache {
|
||||
@@ -64,7 +85,7 @@ impl IndexCache {
|
||||
"IndexCache::new: failed to open layer (partition {p}, layer {l}): {e} — partition is not fully built"
|
||||
)
|
||||
});
|
||||
layers.push(layer);
|
||||
layers.push(Arc::new(layer));
|
||||
}
|
||||
layer_cache.insert(p, layers);
|
||||
}
|
||||
@@ -73,7 +94,7 @@ impl IndexCache {
|
||||
IndexCache {
|
||||
raw_index: index,
|
||||
meta,
|
||||
layer_cache,
|
||||
layer_cache: Arc::new(layer_cache),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,22 +130,52 @@ impl IndexCache {
|
||||
self.layer_cache.keys().copied()
|
||||
}
|
||||
|
||||
/// A cheap `Arc` clone (refcount bump, no reopening) of one cached
|
||||
/// layer — owned, not tied to `&self`'s lifetime, so it can be moved
|
||||
/// into a context that must own it, e.g. a `Send + 'static` pipeline
|
||||
/// worker. See the [`layer_cache`](Self::layer_cache) field docs for
|
||||
/// why every accessor here hands out `Arc<KmerLayer>` rather than
|
||||
/// `&KmerLayer`.
|
||||
#[inline]
|
||||
pub fn get_layer(&self, partition: usize, layer: usize) -> Option<&KmerLayer> {
|
||||
self.layer_cache.get(&partition)?.get(layer)
|
||||
pub fn get_layer(&self, partition: usize, layer: usize) -> Option<Arc<KmerLayer>> {
|
||||
self.layer_cache.get(&partition)?.get(layer).cloned()
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = &KmerLayer> {
|
||||
self.layer_cache.values().flatten()
|
||||
/// Every cached layer, across every cached partition, as owned `Arc`
|
||||
/// clones, in `(partition, layer)` order — genuinely `'static`: see
|
||||
/// [`iter_indexed`](Self::iter_indexed)/[`LayerIter`]'s own docs.
|
||||
pub fn iter(&self) -> impl Iterator<Item = Arc<KmerLayer>> + 'static {
|
||||
self.iter_indexed().map(|(_, _, layer)| layer)
|
||||
}
|
||||
|
||||
pub fn iter_indexed(&self) -> impl Iterator<Item = (usize, usize, &KmerLayer)> {
|
||||
self.layer_cache.iter().flat_map(|(&p, layers)| {
|
||||
layers
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(move |(l, layer)| (p, l, layer))
|
||||
})
|
||||
/// Every cached layer of *one* partition, in layer order — never the
|
||||
/// other cached partitions. For a caller that already knows which
|
||||
/// partition a k-mer routes to (`CanonicalKmer::partition`, the same
|
||||
/// rule [`find`](Self::find) uses) and only needs to scan that one
|
||||
/// partition's own layers, not every partition this cache holds. Empty
|
||||
/// (not an error) if `partition` isn't cached — same convention as
|
||||
/// [`get_layer`](Self::get_layer). `'static` for the same reason
|
||||
/// [`iter_indexed`](Self::iter_indexed) is.
|
||||
pub fn iter_partition(&self, partition: usize) -> impl Iterator<Item = Arc<KmerLayer>> + 'static {
|
||||
LayerIter {
|
||||
cache: Arc::clone(&self.layer_cache),
|
||||
partitions: vec![partition].into_iter(),
|
||||
current: None,
|
||||
}
|
||||
.map(|(_, _, layer)| layer)
|
||||
}
|
||||
|
||||
/// Every cached layer, tagged with its own `(partition, layer)`
|
||||
/// position — see [`LayerIter`]'s own docs for why this is a real
|
||||
/// `next()`-driven iterator, one `Arc<KmerLayer>` clone at a time out
|
||||
/// of the shared cache map, rather than a `Vec<Arc<KmerLayer>>`
|
||||
/// materialised up front.
|
||||
pub fn iter_indexed(&self) -> impl Iterator<Item = (usize, usize, Arc<KmerLayer>)> + 'static {
|
||||
LayerIter {
|
||||
cache: Arc::clone(&self.layer_cache),
|
||||
partitions: self.layer_cache.keys().copied().collect::<Vec<_>>().into_iter(),
|
||||
current: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterate over the unitigs of every cached layer, chained — the
|
||||
@@ -133,7 +184,7 @@ impl IndexCache {
|
||||
/// (one partition, several, or the whole index — see [`new`](Self::new)).
|
||||
/// No extra opening: every layer here is already open, this only
|
||||
/// streams what [`iter`](Self::iter) already gives out.
|
||||
pub fn iter_unitigs(&self) -> impl Iterator<Item = obikseq::Unitig> + '_ {
|
||||
pub fn iter_unitigs(&self) -> impl Iterator<Item = obikseq::Unitig> + 'static {
|
||||
self.iter().flat_map(|l| l.iter_unitigs())
|
||||
}
|
||||
|
||||
@@ -224,7 +275,7 @@ impl IndexCache {
|
||||
.flat_map(|(_, layers)| layers.par_iter())
|
||||
.map(|layer| {
|
||||
let _guard = throttle.acquire_guard();
|
||||
map(layer)
|
||||
map(layer.as_ref())
|
||||
})
|
||||
.reduce(&identity, &reduce)
|
||||
}
|
||||
@@ -244,7 +295,7 @@ impl IndexCache {
|
||||
.flat_map(|(_, layers)| layers.par_iter())
|
||||
.for_each(|layer| {
|
||||
let _guard = throttle.acquire_guard();
|
||||
f(layer);
|
||||
f(layer.as_ref());
|
||||
});
|
||||
}
|
||||
|
||||
@@ -268,8 +319,43 @@ impl IndexCache {
|
||||
.par_iter()
|
||||
.flat_map(|(_, layers)| layers.par_iter())
|
||||
.map(move |layer| obipipeline::Throttled {
|
||||
item: layer,
|
||||
item: layer.as_ref(),
|
||||
guard: throttle.acquire_guard(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A real `Iterator` (implements `next()`), not a `Vec` pre-filled and then
|
||||
/// walked — `next()` looks up and clones exactly one `Arc<KmerLayer>` at a
|
||||
/// time, straight out of `cache` (a clone of `IndexCache`'s own shared map,
|
||||
/// one refcount bump to construct regardless of how many layers are cached),
|
||||
/// so no separate copy of the layers being walked ever exists. `partitions`
|
||||
/// is a small, already-owned `Vec<usize>` of which partition(s) to walk, in
|
||||
/// order — holding a `Vec` of plain partition numbers costs nothing close
|
||||
/// to what holding a `Vec` of the cached layers themselves would.
|
||||
pub struct LayerIter {
|
||||
cache: Arc<HashMap<usize, Vec<Arc<KmerLayer>>>>,
|
||||
partitions: std::vec::IntoIter<usize>,
|
||||
/// `(partition, next layer index)` of the partition currently being
|
||||
/// walked — `None` once that partition is exhausted or before the
|
||||
/// first `next()` call, at which point [`partitions`](Self::partitions)
|
||||
/// is advanced to pick up the next one.
|
||||
current: Option<(usize, usize)>,
|
||||
}
|
||||
|
||||
impl Iterator for LayerIter {
|
||||
type Item = (usize, usize, Arc<KmerLayer>);
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
loop {
|
||||
if let Some((partition, layer_idx)) = self.current {
|
||||
if let Some(layer) = self.cache.get(&partition).and_then(|v| v.get(layer_idx)) {
|
||||
self.current = Some((partition, layer_idx + 1));
|
||||
return Some((partition, layer_idx, Arc::clone(layer)));
|
||||
}
|
||||
self.current = None; // this partition exhausted
|
||||
}
|
||||
self.current = Some((self.partitions.next()?, 0));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user