Fix batch enumeration offsets and refactor sibling annex construction

Shifts sibling annex construction from slot-indexed enumeration to iteration-order traversal by correcting cumulative k-mer offset tracking in batch enumeration. Replaces coarse per-partition parallelism with chunked work distribution to prevent thread starvation on skewed partitions. Decouples custom progress messages from ETA updates to eliminate display clobbering during high-frequency callbacks. Adds regression tests validating batch offset correctness, partial batch handling, and iterator-order consistency across layer builds.
This commit is contained in:
Eric Coissac
2026-08-16 20:51:19 +02:00
parent d2548e8c33
commit 32d6720f50
11 changed files with 413 additions and 88 deletions
+1
View File
@@ -20,3 +20,4 @@ pub use meta::{validate_label, GenomeInfo, IndexConfig, IndexMeta, META_FILENAME
pub use predicate::{GroupFilterParams, MetaPred};
pub use state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED};
pub use stats::IndexBitsPerKmer;
pub use numa::PartitionRunner;
+26
View File
@@ -82,6 +82,32 @@ impl PartitionRunner {
Self { nodes }
}
/// Like [`new`](Self::new), but caps total worker slots (summed across
/// nodes) at `max_total_workers` — split evenly across nodes, each
/// further capped by that node's actual core count. For callers whose
/// own per-worker closure does further internal parallel work (so the
/// natural per-node core count would oversubscribe if used as the
/// *outer* degree of parallelism too).
pub fn new_capped(max_total_workers: usize) -> Self {
let ns = build();
let n_nodes = ns.pools.len().max(1);
let per_node_cap = (max_total_workers / n_nodes).max(1);
debug!(
"PartitionRunner (capped): {} node(s) × up to {} worker(s)/node ({} total requested)",
n_nodes, per_node_cap, max_total_workers,
);
let nodes = ns
.pools
.into_iter()
.zip(ns.cpus_per_node)
.map(|(pool, cpu_ids)| {
let node_cores = cpu_ids.len().max(1);
NodeConfig { pool, cpu_ids, max_workers: per_node_cap.min(node_cores) }
})
.collect();
Self { nodes }
}
/// Run `f(i)` for every index in `order`.
///
/// Workers are pre-spawned dormant and activated adaptively, per node: