Fix batch enumeration offsets and refactor sibling annex construction
Shifts sibling annex construction from slot-indexed enumeration to iteration-order traversal by correcting cumulative k-mer offset tracking in batch enumeration. Replaces coarse per-partition parallelism with chunked work distribution to prevent thread starvation on skewed partitions. Decouples custom progress messages from ETA updates to eliminate display clobbering during high-frequency callbacks. Adds regression tests validating batch offset correctness, partial batch handling, and iterator-order consistency across layer builds.
This commit is contained in:
@@ -20,3 +20,4 @@ pub use meta::{validate_label, GenomeInfo, IndexConfig, IndexMeta, META_FILENAME
|
||||
pub use predicate::{GroupFilterParams, MetaPred};
|
||||
pub use state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED};
|
||||
pub use stats::IndexBitsPerKmer;
|
||||
pub use numa::PartitionRunner;
|
||||
|
||||
@@ -82,6 +82,32 @@ impl PartitionRunner {
|
||||
Self { nodes }
|
||||
}
|
||||
|
||||
/// Like [`new`](Self::new), but caps total worker slots (summed across
|
||||
/// nodes) at `max_total_workers` — split evenly across nodes, each
|
||||
/// further capped by that node's actual core count. For callers whose
|
||||
/// own per-worker closure does further internal parallel work (so the
|
||||
/// natural per-node core count would oversubscribe if used as the
|
||||
/// *outer* degree of parallelism too).
|
||||
pub fn new_capped(max_total_workers: usize) -> Self {
|
||||
let ns = build();
|
||||
let n_nodes = ns.pools.len().max(1);
|
||||
let per_node_cap = (max_total_workers / n_nodes).max(1);
|
||||
debug!(
|
||||
"PartitionRunner (capped): {} node(s) × up to {} worker(s)/node ({} total requested)",
|
||||
n_nodes, per_node_cap, max_total_workers,
|
||||
);
|
||||
let nodes = ns
|
||||
.pools
|
||||
.into_iter()
|
||||
.zip(ns.cpus_per_node)
|
||||
.map(|(pool, cpu_ids)| {
|
||||
let node_cores = cpu_ids.len().max(1);
|
||||
NodeConfig { pool, cpu_ids, max_workers: per_node_cap.min(node_cores) }
|
||||
})
|
||||
.collect();
|
||||
Self { nodes }
|
||||
}
|
||||
|
||||
/// Run `f(i)` for every index in `order`.
|
||||
///
|
||||
/// Workers are pre-spawned dormant and activated adaptively, per node:
|
||||
|
||||
Reference in New Issue
Block a user