feat: add performance instrumentation and dynamic worker scaling

This change enhances observability and adaptability in the merge pipeline. Performance timing and debug logging are added to the De Bruijn graph and partition merge layers to track phase durations and pipeline metrics. The merge module replaces blocking receives with timed polls to sample CPU efficiency, dynamically spawning workers when utilization drops below a threshold. A new script is also introduced to parse merge debug logs and generate structured Markdown reports detailing throughput, phase breakdowns, and partition performance.

src/obikindex/src/merge.rs

@@ -289,14 +289,39 @@ impl KmerIndex {
                 activate_tx.send(()).ok();
                 n_workers = 1;
 
+                const SPAWN_POLL: Duration = Duration::from_secs(10);
+
                 let mut completed = 0usize;
                 while completed < n_partitions {
-                    let (i, r, dur) = result_rx.recv().map_err(|_| {
-                        OKIError::Io(io::Error::new(
-                            io::ErrorKind::UnexpectedEof,
-                            "worker channel closed",
-                        ))
-                    })?;
+                    let result = result_rx.recv_timeout(SPAWN_POLL);
+
+                    // On timeout: no partition finished yet, just check efficiency.
+                    let (i, r, dur) = match result {
+                        Ok(v) => v,
+                        Err(crossbeam_channel::RecvTimeoutError::Timeout) => {
+                            if n_workers < max_workers {
+                                let eff = cpu_sample.cpu_efficiency(n_cores);
+                                if eff < SPAWN_THRESHOLD {
+                                    debug!(
+                                        "activated worker {} (poll) — efficiency {:.0}%",
+                                        n_workers + 1,
+                                        eff * 100.0,
+                                    );
+                                    efficiency_at_last_spawn = eff;
+                                    activate_tx.send(()).ok();
+                                    n_workers += 1;
+                                    cpu_sample = CpuSample::now();
+                                }
+                            }
+                            continue;
+                        }
+                        Err(crossbeam_channel::RecvTimeoutError::Disconnected) => {
+                            return Err(OKIError::Io(io::Error::new(
+                                io::ErrorKind::UnexpectedEof,
+                                "worker channel closed",
+                            )));
+                        }
+                    };
                     let g_len = r.map_err(OKIError::Partition)?;
                     pb.inc(1);
                     debug!(