feat: add performance instrumentation and dynamic worker scaling

This change enhances observability and adaptability in the merge pipeline. Performance timing and debug logging are added to the De Bruijn graph and partition merge layers to track phase durations and pipeline metrics. The merge module replaces blocking receives with timed polls to sample CPU efficiency, dynamically spawning workers when utilization drops below a threshold. A new script is also introduced to parse merge debug logs and generate structured Markdown reports detailing throughput, phase breakdowns, and partition performance.

src/obikpartitionner/src/merge_layer.rs

@@ -304,27 +304,37 @@ impl KmerPartition {
         let new_layer_dir = dst_index_dir.join(format!("layer_{new_layer_idx}"));
 
         let n_new = if any_new {
+            let t_deg = std::time::Instant::now();
             g.compute_degrees_and_mark_starts();
+            debug!("partition {i}: compute_degrees in {:.3}s — {} nodes",
+                t_deg.elapsed().as_secs_f64(), g.len());
             fs::create_dir_all(&new_layer_dir)?;
             let mut uw = Layer::<()>::unitig_writer(&new_layer_dir).map_err(olm_to_sk)?;
+            debug!("partition {i}: unitig traversal start — {} nodes", g.len());
             g.try_for_each_unitig(|unitig| {
                 uw.write(unitig)
             })?;
+            debug!("partition {i}: unitig writer closing");
             uw.close()?;
+            debug!("partition {i}: unitig writer closed — dropping graph ({} nodes)", g.len());
             let n = g.len();
-            drop(g); // release GraphDeBruijn before MPHF build
+            drop(g);
+            debug!("partition {i}: graph dropped — starting MPHF build ({n} unitigs)");
             Layer::<()>::build(&new_layer_dir, block_bits, evidence).map_err(olm_to_sk)?;
+            debug!("partition {i}: MPHF build done");
             n
         } else {
             drop(g);
             0
         };
 
+        let t_open = std::time::Instant::now();
         let new_mphf: Option<Arc<MphfOnly>> = if any_new {
             Some(Arc::new(MphfOnly::open(&new_layer_dir).map_err(olm_to_sk)?))
         } else {
             None
         };
+        debug!("partition {i}: MPHF open in {:.3}s", t_open.elapsed().as_secs_f64());
 
         // ── Prepare matrix directories for the new layer ──────────────────────
         // Absent columns (dst genomes) are written via append_column (all-zero/false).
@@ -379,6 +389,7 @@ impl KmerPartition {
             vec![]
         };
 
+        let t_builders = std::time::Instant::now();
         // Builders for existing layers: n_src_total per layer.
         // Columns n_dst_genomes .. n_dst_genomes + n_src_total - 1.
         let exist_builders: Vec<Vec<ColBuilder>> = (0..n_dst_layers)
@@ -410,7 +421,10 @@ impl KmerPartition {
             })
             .collect::<SKResult<_>>()?;
 
+        debug!("partition {i}: builders ready in {:.3}s", t_builders.elapsed().as_secs_f64());
+
         // ── Pass 2: fill builders (pipeline) ─────────────────────────────────
+        let t_pass2 = std::time::Instant::now();
         // Collect source items before the pipeline so load_meta errors propagate
         // via ? before any worker thread is spawned.
         let mut pass2_items: Vec<(usize, usize, PathBuf)> = Vec::new();
@@ -531,6 +545,7 @@ impl KmerPartition {
         );
 
         WorkerPool::new(pipeline2, n_workers, capacity).run();
+        debug!("partition {i}: pass2 pipeline done in {:.3}s", t_pass2.elapsed().as_secs_f64());
 
         if let Some(msg) = Arc::try_unwrap(pass2_err)
             .unwrap_or_else(|_| panic!("pass2: pass2_err not uniquely owned"))
@@ -545,6 +560,7 @@ impl KmerPartition {
             .into_inner()
             .unwrap_or_else(|e| e.into_inner());
 
+        let t_close = std::time::Instant::now();
         // ── Close builders and update metadata ────────────────────────────────
         for (l, builders) in exist_builders.into_iter().enumerate() {
             let layer_dir = dst_index_dir.join(format!("layer_{l}"));
@@ -575,6 +591,8 @@ impl KmerPartition {
             part_meta.save(&dst_index_dir).map_err(olm_to_sk)?;
         }
 
+        debug!("partition {i}: builders closed in {:.3}s", t_close.elapsed().as_secs_f64());
+
         Ok(n_new)
     }
 }