Refactor sampling algorithms and coverage metrics

Optimize family scanning and sampling plan calculation logic. Introduce new per-genome richness metrics and redefine the sampling fraction $r$ for improved analysis.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:01:15 +02:00
1 parent 0e40e83fd1
commit c226744405
3 files changed
+70 -10

No files matched your search

@@ -229,6 +229,9 @@ pub(crate) fn scan_layer_families(
let mut genome_mask = vec![0u8; n * ctx.n_genomes];
let mut outgoing: Vec<Vec<(CanonicalKmer, usize, u8, u8)>> =
(0..ctx.n_parts).map(|_| Vec::new()).collect();
// Batch positions of the families `selected` keeps — the only
// ones pass 2 reads the matrix for.
let mut picked: Vec<usize> = Vec::new();
// Pass 1: cheap, no matrix access — own base and this
// batch's cross-partition queries. Kmer and mask already in
@@ -239,8 +242,9 @@ pub(crate) fn scan_layer_families(
// hands back each one's annex-recorded layer alongside.
// Families outside `selected` (sampling only) still get a
// mask/base entry — pass 2 indexes uniformly by `i` — but
// never an outgoing query: that's the expensive part a
// sampled run exists to avoid paying for every family.
// are neither queried across partitions nor read from the
// matrix in pass 2: that's the expensive part a sampled run
// exists to avoid paying for every family.
for (i, entry) in batch.entries.iter().enumerate() {
let (kmer, mask) = (entry.kmer, entry.mask);
masks.push(mask);
@@ -250,6 +254,7 @@ pub(crate) fn scan_layer_families(
if !is_selected(&worker_selected, family_idx) {
continue;
}
picked.push(i);
for (member, layer) in mask.family_members(kmer, ctx.k) {
if member == kmer {
continue; // local — resolved below straight from `mat`, no lookup
@@ -269,14 +274,22 @@ pub(crate) fn scan_layer_families(
// each entry's kmer is mapped to its slot via `hash_batch`
// — a pure MPHF lookup, no evidence check, since these are
// this layer's own kmers, known members by construction.
let kmers: Vec<CanonicalKmer> = batch.entries.iter().map(|e| e.kmer).collect();
let slots = ctx.mat.hash_batch(&kmers);
let mut carries: Vec<Vec<bool>> = (0..ctx.n_cols).map(|_| Vec::new()).collect();
ctx.mat.fill_sub_matrix_carries(&slots, &mut carries);
for (g, col) in carries.iter().enumerate() {
for (i, &carries_it) in col.iter().enumerate() {
if carries_it {
genome_mask[i * ctx.n_genomes + g] |= 1 << bases[i];
// Only the `picked` families are read: a sampled or
// annex-restricted scan visits a handful of a layer's
// minorants, and a full-batch matrix pass would cost as much
// as `Selection::All`. `carries[g][j]` belongs to batch
// position `picked[j]`.
if !picked.is_empty() {
let kmers: Vec<CanonicalKmer> =
picked.iter().map(|&i| batch.entries[i].kmer).collect();
let slots = ctx.mat.hash_batch(&kmers);
let mut carries: Vec<Vec<bool>> = (0..ctx.n_cols).map(|_| Vec::new()).collect();
ctx.mat.fill_sub_matrix_carries(&slots, &mut carries);
for (g, col) in carries.iter().enumerate() {
for (&i, &carries_it) in picked.iter().zip(col) {
if carries_it {
genome_mask[i * ctx.n_genomes + g] |= 1 << bases[i];
}
}
}
}
@@ -49,6 +49,7 @@ use std::sync::Arc;
use obicompactvec::TempBitVecBuilder;
use obikidxcache::index_cache::IndexCache;
use obikindex::OKIResult;
use obisys::progress_bar;
use rand::RngExt;
use super::family_scan::{Selection, scan_layer_families};
@@ -263,6 +264,7 @@ pub(crate) fn sample_index(
// — needed before any layer's quota can be computed (each layer's share
// depends on every other layer's own mass).
let mut plans: Vec<LayerPlan> = Vec::new();
let pb = progress_bar("sampling_plan", cache.n_partition() as u64, "partitions");
for partition in cache.partitions() {
let n_layer = cache.n_layer(partition).unwrap_or(0);
for layer_idx in 0..n_layer {
@@ -293,7 +295,9 @@ pub(crate) fn sample_index(
mass,
});
}
pb.inc(1);
}
pb.finish_and_clear();
let total_mass: f64 = plans.iter().map(|plan| plan.mass).sum();
let mut report = SamplePlan { target: n, layers: Vec::with_capacity(plans.len()) };
@@ -308,6 +312,7 @@ pub(crate) fn sample_index(
// them, when `n` is smaller than the number of layers.
let mut cumulative_mass = 0.0f64;
let mut allotted = 0usize;
let pb = progress_bar("sampling", plans.len() as u64, "layers");
for (raw_index, mut plan) in plans.into_iter().enumerate() {
cumulative_mass += plan.mass;
let cumulative_quota = (n as f64 * cumulative_mass / total_mass) as usize;
@@ -337,7 +342,9 @@ pub(crate) fn sample_index(
resolved,
kept,
});
pb.inc(1);
}
pb.finish_and_clear();
Ok(report)
}