Refactor sampling algorithms and coverage metrics
Optimize family scanning and sampling plan calculation logic. Introduce new per-genome richness metrics and redefine the sampling fraction $r$ for improved analysis.
This commit is contained in:
1 parent
0e40e83fd1
commit
c226744405
3 files changed
+70
-10
No files matched your search
@@ -229,6 +229,9 @@ pub(crate) fn scan_layer_families(
|
||||
let mut genome_mask = vec![0u8; n * ctx.n_genomes];
|
||||
let mut outgoing: Vec<Vec<(CanonicalKmer, usize, u8, u8)>> =
|
||||
(0..ctx.n_parts).map(|_| Vec::new()).collect();
|
||||
// Batch positions of the families `selected` keeps — the only
|
||||
// ones pass 2 reads the matrix for.
|
||||
let mut picked: Vec<usize> = Vec::new();
|
||||
|
||||
// Pass 1: cheap, no matrix access — own base and this
|
||||
// batch's cross-partition queries. Kmer and mask already in
|
||||
@@ -239,8 +242,9 @@ pub(crate) fn scan_layer_families(
|
||||
// hands back each one's annex-recorded layer alongside.
|
||||
// Families outside `selected` (sampling only) still get a
|
||||
// mask/base entry — pass 2 indexes uniformly by `i` — but
|
||||
// never an outgoing query: that's the expensive part a
|
||||
// sampled run exists to avoid paying for every family.
|
||||
// are neither queried across partitions nor read from the
|
||||
// matrix in pass 2: that's the expensive part a sampled run
|
||||
// exists to avoid paying for every family.
|
||||
for (i, entry) in batch.entries.iter().enumerate() {
|
||||
let (kmer, mask) = (entry.kmer, entry.mask);
|
||||
masks.push(mask);
|
||||
@@ -250,6 +254,7 @@ pub(crate) fn scan_layer_families(
|
||||
if !is_selected(&worker_selected, family_idx) {
|
||||
continue;
|
||||
}
|
||||
picked.push(i);
|
||||
for (member, layer) in mask.family_members(kmer, ctx.k) {
|
||||
if member == kmer {
|
||||
continue; // local — resolved below straight from `mat`, no lookup
|
||||
@@ -269,14 +274,22 @@ pub(crate) fn scan_layer_families(
|
||||
// each entry's kmer is mapped to its slot via `hash_batch`
|
||||
// — a pure MPHF lookup, no evidence check, since these are
|
||||
// this layer's own kmers, known members by construction.
|
||||
let kmers: Vec<CanonicalKmer> = batch.entries.iter().map(|e| e.kmer).collect();
|
||||
let slots = ctx.mat.hash_batch(&kmers);
|
||||
let mut carries: Vec<Vec<bool>> = (0..ctx.n_cols).map(|_| Vec::new()).collect();
|
||||
ctx.mat.fill_sub_matrix_carries(&slots, &mut carries);
|
||||
for (g, col) in carries.iter().enumerate() {
|
||||
for (i, &carries_it) in col.iter().enumerate() {
|
||||
if carries_it {
|
||||
genome_mask[i * ctx.n_genomes + g] |= 1 << bases[i];
|
||||
// Only the `picked` families are read: a sampled or
|
||||
// annex-restricted scan visits a handful of a layer's
|
||||
// minorants, and a full-batch matrix pass would cost as much
|
||||
// as `Selection::All`. `carries[g][j]` belongs to batch
|
||||
// position `picked[j]`.
|
||||
if !picked.is_empty() {
|
||||
let kmers: Vec<CanonicalKmer> =
|
||||
picked.iter().map(|&i| batch.entries[i].kmer).collect();
|
||||
let slots = ctx.mat.hash_batch(&kmers);
|
||||
let mut carries: Vec<Vec<bool>> = (0..ctx.n_cols).map(|_| Vec::new()).collect();
|
||||
ctx.mat.fill_sub_matrix_carries(&slots, &mut carries);
|
||||
for (g, col) in carries.iter().enumerate() {
|
||||
for (&i, &carries_it) in picked.iter().zip(col) {
|
||||
if carries_it {
|
||||
genome_mask[i * ctx.n_genomes + g] |= 1 << bases[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +49,7 @@ use std::sync::Arc;
|
||||
use obicompactvec::TempBitVecBuilder;
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::OKIResult;
|
||||
use obisys::progress_bar;
|
||||
use rand::RngExt;
|
||||
|
||||
use super::family_scan::{Selection, scan_layer_families};
|
||||
@@ -263,6 +264,7 @@ pub(crate) fn sample_index(
|
||||
// — needed before any layer's quota can be computed (each layer's share
|
||||
// depends on every other layer's own mass).
|
||||
let mut plans: Vec<LayerPlan> = Vec::new();
|
||||
let pb = progress_bar("sampling_plan", cache.n_partition() as u64, "partitions");
|
||||
for partition in cache.partitions() {
|
||||
let n_layer = cache.n_layer(partition).unwrap_or(0);
|
||||
for layer_idx in 0..n_layer {
|
||||
@@ -293,7 +295,9 @@ pub(crate) fn sample_index(
|
||||
mass,
|
||||
});
|
||||
}
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
let total_mass: f64 = plans.iter().map(|plan| plan.mass).sum();
|
||||
let mut report = SamplePlan { target: n, layers: Vec::with_capacity(plans.len()) };
|
||||
@@ -308,6 +312,7 @@ pub(crate) fn sample_index(
|
||||
// them, when `n` is smaller than the number of layers.
|
||||
let mut cumulative_mass = 0.0f64;
|
||||
let mut allotted = 0usize;
|
||||
let pb = progress_bar("sampling", plans.len() as u64, "layers");
|
||||
for (raw_index, mut plan) in plans.into_iter().enumerate() {
|
||||
cumulative_mass += plan.mass;
|
||||
let cumulative_quota = (n as f64 * cumulative_mass / total_mass) as usize;
|
||||
@@ -337,7 +342,9 @@ pub(crate) fn sample_index(
|
||||
resolved,
|
||||
kept,
|
||||
});
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user