feat: add performance instrumentation and dynamic worker scaling
This change enhances observability and adaptability in the merge pipeline. Performance timing and debug logging are added to the De Bruijn graph and partition merge layers to track phase durations and pipeline metrics. The merge module replaces blocking receives with timed polls to sample CPU efficiency, dynamically spawning workers when utilization drops below a threshold. A new script is also introduced to parse merge debug logs and generate structured Markdown reports detailing throughput, phase breakdowns, and partition performance.
This commit is contained in:
Eric Coissac
@@ -7,8 +7,6 @@ use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
|
||||
use std::cell::RefCell;
|
||||
use std::fmt;
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
use std::time::Instant;
|
||||
use tracing::{debug, info};
|
||||
use xxhash_rust::xxh3::Xxh3Builder;
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
@@ -285,7 +283,6 @@ impl GraphDeBruijn {
|
||||
pub fn compute_degrees_and_mark_starts(&self) {
|
||||
// Pass 1: count right/left neighbors for each node
|
||||
|
||||
let t1 = Instant::now();
|
||||
self.for_each_node(|kmer, atomic| {
|
||||
let mut old = Node(atomic.load(Ordering::Relaxed));
|
||||
if old.is_visited() {
|
||||
@@ -295,20 +292,13 @@ impl GraphDeBruijn {
|
||||
}
|
||||
let (rc, rn) = count_neighbors(&kmer.right_canonical_neighbors(), &self.nodes);
|
||||
let (lc, ln) = count_neighbors(&kmer.left_canonical_neighbors(), &self.nodes);
|
||||
let mut node = Node(0); // reset all bits (visited=0, start=0)
|
||||
let mut node = Node(0);
|
||||
node.set_right(rc, rn);
|
||||
node.set_left(lc, ln);
|
||||
atomic.store(node.0, Ordering::Relaxed);
|
||||
});
|
||||
debug!(
|
||||
"[compute_degrees] pass 1 (degrees): {:?} — {} nodes",
|
||||
t1.elapsed(),
|
||||
self.nodes.len()
|
||||
);
|
||||
|
||||
// Pass 2: mark start nodes
|
||||
|
||||
let t2 = Instant::now();
|
||||
self.for_each_node(|kmer, atomic| {
|
||||
let mut node = Node(atomic.load(Ordering::Relaxed));
|
||||
if node.is_visited() {
|
||||
@@ -319,11 +309,6 @@ impl GraphDeBruijn {
|
||||
atomic.store(node.0, Ordering::Relaxed);
|
||||
}
|
||||
});
|
||||
debug!(
|
||||
"[compute_degrees] pass 2 (starts): {:?} — {} nodes",
|
||||
t2.elapsed(),
|
||||
self.nodes.len()
|
||||
);
|
||||
}
|
||||
|
||||
pub fn is_visited(&self, kmer: &CanonicalKmer) -> Option<bool> {
|
||||
@@ -391,7 +376,6 @@ impl GraphDeBruijn {
|
||||
let n2 = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
// Boucle unique : traiter les starts, recalculer les arités, recommencer
|
||||
let mut pass = 0usize;
|
||||
loop {
|
||||
let n_new = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
@@ -421,9 +405,7 @@ impl GraphDeBruijn {
|
||||
});
|
||||
|
||||
let n = n_new.load(Ordering::Relaxed);
|
||||
debug!("[for_each_unitig] pass {}: {} starts", pass, n);
|
||||
n_chains.fetch_add(n, Ordering::Relaxed);
|
||||
pass += 1;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
@@ -452,12 +434,6 @@ impl GraphDeBruijn {
|
||||
}
|
||||
}
|
||||
|
||||
debug!(
|
||||
chains = n_chains.load(Ordering::Relaxed),
|
||||
phase2 = n2.load(Ordering::Relaxed),
|
||||
total = n_chains.load(Ordering::Relaxed) + n2.load(Ordering::Relaxed),
|
||||
"unitig traversal complete"
|
||||
);
|
||||
}
|
||||
|
||||
/// Merge `other` into `self`.
|
||||
|
||||
@@ -289,14 +289,39 @@ impl KmerIndex {
|
||||
activate_tx.send(()).ok();
|
||||
n_workers = 1;
|
||||
|
||||
const SPAWN_POLL: Duration = Duration::from_secs(10);
|
||||
|
||||
let mut completed = 0usize;
|
||||
while completed < n_partitions {
|
||||
let (i, r, dur) = result_rx.recv().map_err(|_| {
|
||||
OKIError::Io(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
"worker channel closed",
|
||||
))
|
||||
})?;
|
||||
let result = result_rx.recv_timeout(SPAWN_POLL);
|
||||
|
||||
// On timeout: no partition finished yet, just check efficiency.
|
||||
let (i, r, dur) = match result {
|
||||
Ok(v) => v,
|
||||
Err(crossbeam_channel::RecvTimeoutError::Timeout) => {
|
||||
if n_workers < max_workers {
|
||||
let eff = cpu_sample.cpu_efficiency(n_cores);
|
||||
if eff < SPAWN_THRESHOLD {
|
||||
debug!(
|
||||
"activated worker {} (poll) — efficiency {:.0}%",
|
||||
n_workers + 1,
|
||||
eff * 100.0,
|
||||
);
|
||||
efficiency_at_last_spawn = eff;
|
||||
activate_tx.send(()).ok();
|
||||
n_workers += 1;
|
||||
cpu_sample = CpuSample::now();
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(crossbeam_channel::RecvTimeoutError::Disconnected) => {
|
||||
return Err(OKIError::Io(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
"worker channel closed",
|
||||
)));
|
||||
}
|
||||
};
|
||||
let g_len = r.map_err(OKIError::Partition)?;
|
||||
pb.inc(1);
|
||||
debug!(
|
||||
|
||||
@@ -304,27 +304,37 @@ impl KmerPartition {
|
||||
let new_layer_dir = dst_index_dir.join(format!("layer_{new_layer_idx}"));
|
||||
|
||||
let n_new = if any_new {
|
||||
let t_deg = std::time::Instant::now();
|
||||
g.compute_degrees_and_mark_starts();
|
||||
debug!("partition {i}: compute_degrees in {:.3}s — {} nodes",
|
||||
t_deg.elapsed().as_secs_f64(), g.len());
|
||||
fs::create_dir_all(&new_layer_dir)?;
|
||||
let mut uw = Layer::<()>::unitig_writer(&new_layer_dir).map_err(olm_to_sk)?;
|
||||
debug!("partition {i}: unitig traversal start — {} nodes", g.len());
|
||||
g.try_for_each_unitig(|unitig| {
|
||||
uw.write(unitig)
|
||||
})?;
|
||||
debug!("partition {i}: unitig writer closing");
|
||||
uw.close()?;
|
||||
debug!("partition {i}: unitig writer closed — dropping graph ({} nodes)", g.len());
|
||||
let n = g.len();
|
||||
drop(g); // release GraphDeBruijn before MPHF build
|
||||
drop(g);
|
||||
debug!("partition {i}: graph dropped — starting MPHF build ({n} unitigs)");
|
||||
Layer::<()>::build(&new_layer_dir, block_bits, evidence).map_err(olm_to_sk)?;
|
||||
debug!("partition {i}: MPHF build done");
|
||||
n
|
||||
} else {
|
||||
drop(g);
|
||||
0
|
||||
};
|
||||
|
||||
let t_open = std::time::Instant::now();
|
||||
let new_mphf: Option<Arc<MphfOnly>> = if any_new {
|
||||
Some(Arc::new(MphfOnly::open(&new_layer_dir).map_err(olm_to_sk)?))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
debug!("partition {i}: MPHF open in {:.3}s", t_open.elapsed().as_secs_f64());
|
||||
|
||||
// ── Prepare matrix directories for the new layer ──────────────────────
|
||||
// Absent columns (dst genomes) are written via append_column (all-zero/false).
|
||||
@@ -379,6 +389,7 @@ impl KmerPartition {
|
||||
vec![]
|
||||
};
|
||||
|
||||
let t_builders = std::time::Instant::now();
|
||||
// Builders for existing layers: n_src_total per layer.
|
||||
// Columns n_dst_genomes .. n_dst_genomes + n_src_total - 1.
|
||||
let exist_builders: Vec<Vec<ColBuilder>> = (0..n_dst_layers)
|
||||
@@ -410,7 +421,10 @@ impl KmerPartition {
|
||||
})
|
||||
.collect::<SKResult<_>>()?;
|
||||
|
||||
debug!("partition {i}: builders ready in {:.3}s", t_builders.elapsed().as_secs_f64());
|
||||
|
||||
// ── Pass 2: fill builders (pipeline) ─────────────────────────────────
|
||||
let t_pass2 = std::time::Instant::now();
|
||||
// Collect source items before the pipeline so load_meta errors propagate
|
||||
// via ? before any worker thread is spawned.
|
||||
let mut pass2_items: Vec<(usize, usize, PathBuf)> = Vec::new();
|
||||
@@ -531,6 +545,7 @@ impl KmerPartition {
|
||||
);
|
||||
|
||||
WorkerPool::new(pipeline2, n_workers, capacity).run();
|
||||
debug!("partition {i}: pass2 pipeline done in {:.3}s", t_pass2.elapsed().as_secs_f64());
|
||||
|
||||
if let Some(msg) = Arc::try_unwrap(pass2_err)
|
||||
.unwrap_or_else(|_| panic!("pass2: pass2_err not uniquely owned"))
|
||||
@@ -545,6 +560,7 @@ impl KmerPartition {
|
||||
.into_inner()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
|
||||
let t_close = std::time::Instant::now();
|
||||
// ── Close builders and update metadata ────────────────────────────────
|
||||
for (l, builders) in exist_builders.into_iter().enumerate() {
|
||||
let layer_dir = dst_index_dir.join(format!("layer_{l}"));
|
||||
@@ -575,6 +591,8 @@ impl KmerPartition {
|
||||
part_meta.save(&dst_index_dir).map_err(olm_to_sk)?;
|
||||
}
|
||||
|
||||
debug!("partition {i}: builders closed in {:.3}s", t_close.elapsed().as_secs_f64());
|
||||
|
||||
Ok(n_new)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user