feat: add pack optimization prompt, debug infrastructure, and monitoring scripts

Add a prompt template for optimizing the `pack` step and a suite of Docker-based
debug tools including build, run, and monitoring scripts. Enhance `pack_matrices`
with LFD partition sizing and detailed timing reports, and update `stop` with
additional system statistics.

- Add `prompts/pack_optimization.md` to guide optimization of the `pack` step,
  specifying context, performance measurements, and deliverable structures.
- Add `scripts/docker_debug/` infrastructure: Dockerfile for musl targets,
  `build.sh` for image creation, `run.sh` for parallel indexing and merging,
  `monitor.sh` for PID tracking, `gen_fasta.py` for synthetic genomes, and
  `gen_fasta_real.py` for real genome generation with `read_fna`.
- Add `scripts/monitor_pack.sh` as a 1 Hz sampler for I/O and memory metrics,
  writing samples to CSV and summary reports.
- Add `scripts/watch_merge.sh` to monitor process resource usage, enforcing
  memory limits via SIGTERM/SIGKILL guards based on VmRSS and VmSize.
- Modify `src/obikindex/src/index/kmer_index.rs`: add `pending_matrix_bytes` and
  `report_pack_timings`; update `pack_matrices` to use LFD strategy for partition
  sizing and clarify documentation regarding `stat()` proxy bytes.
- Modify `src/obisys/src/stage.rs`: update `stop` definition to include new
  variables (`user_secs`, `sys_secs`, faults, context switches, I/O blocks) and
  logic checking for swap usage.
This commit is contained in:
Eric Coissac committed 2026-11-19 07:07:50 +01:00
1 parent 24291a4484
commit 03eaa4544a
11 files changed
+497 -2

No files matched your search

+76 -2
View File
@@ -296,7 +296,14 @@ impl KmerIndex {
};
let n = self.n_partitions();
let order: Vec<usize> = (0..n).collect();
// Largest partitions first (LFD), so the long jobs do not end up as
// stragglers at the tail. Sizes are `stat()` only; the sort is stable,
// ties keep partition-number order. Partitions are independent, so
// the order has no effect on what is written.
let sizes: Vec<u64> = (0..n).map(|i| self.pending_matrix_bytes(i)).collect();
let mut order: Vec<usize> = (0..n).collect();
order.sort_by_key(|&i| std::cmp::Reverse(sizes[i]));
let mut timings: Vec<(usize, u64, std::time::Duration)> = Vec::with_capacity(n);
let pb = progress_bar("pack", n as u64, "partitions");
obisys::PartitionRunner::new().run(
&order,
@@ -327,14 +334,50 @@ impl KmerIndex {
}
Ok(())
},
|_, _, _| {
|i, _, elapsed| {
tracing::debug!(
partition = i,
matrix_bytes = sizes[i],
secs = elapsed.as_secs_f64(),
"pack: partition done"
);
timings.push((i, sizes[i], elapsed));
pb.inc(1);
},
)?;
pb.finish_and_clear();
report_pack_timings(&timings);
Ok(())
}
/// Bytes of matrix data (`presence/` and `counts/` of every layer) still
/// present in partition `i`, from `stat()` only. Before packing this is
/// the columnar data, i.e. a proxy for the work `pack_matrices` has to do
/// on that partition. Missing partition or directories count as 0.
pub fn pending_matrix_bytes(&self, i: usize) -> u64 {
let Ok(n_layers) = self.n_layers(i) else {
return 0;
};
let mut total = 0u64;
for l in 0..n_layers {
let Ok(layer_dir) = self.layer_dir(i, l) else {
continue;
};
for sub in ["presence", "counts"] {
let Ok(entries) = fs::read_dir(layer_dir.join(sub)) else {
continue;
};
total += entries
.flatten()
.filter_map(|e| e.metadata().ok())
.filter(|m| m.is_file())
.map(|m| m.len())
.sum::<u64>();
}
}
total
}
// /// Write a `layer_meta.json` in any layer directory that is missing one.
// ///
// /// Old indexes were built before this file was required. The number of
@@ -394,3 +437,34 @@ impl KmerIndex {
// Ok(())
// }
}
/// One `info!` summary of `pack_matrices` per-partition timings: the sum of
/// partition times (divide by the stage wall time to get the mean number of
/// concurrently busy workers), the median and maximum, and the slowest
/// partitions with their pre-pack matrix size. `timings` holds
/// `(partition, matrix_bytes, elapsed)`.
fn report_pack_timings(timings: &[(usize, u64, std::time::Duration)]) {
if timings.is_empty() {
return;
}
let mut by_time: Vec<&(usize, u64, std::time::Duration)> = timings.iter().collect();
by_time.sort_by_key(|t| std::cmp::Reverse(t.2));
let sum: f64 = timings.iter().map(|t| t.2.as_secs_f64()).sum();
let bytes: u64 = timings.iter().map(|t| t.1).sum();
let median = by_time[by_time.len() / 2].2.as_secs_f64();
tracing::info!(
"pack: {} partition(s), sum of partition times {:.0}s, median {:.1}s, max {:.1}s, {:.1} GB of matrix data",
timings.len(),
sum,
median,
by_time[0].2.as_secs_f64(),
bytes as f64 / 1e9,
);
for (i, b, d) in by_time.iter().take(5).map(|t| (t.0, t.1, t.2)) {
tracing::info!(
"pack: slowest partition {i}: {:.1}s, {:.2} GB of matrix data",
d.as_secs_f64(),
b as f64 / 1e9,
);
}
}
+8
View File
@@ -49,6 +49,14 @@ impl Stage {
wall_secs = format_args!("{:.3}", stats.wall_secs),
rss = %fmt_bytes(stats.max_rss_bytes),
swaps = stats.swaps,
user_secs = format_args!("{:.1}", stats.user_secs),
sys_secs = format_args!("{:.1}", stats.sys_secs),
minor_faults = stats.minor_faults,
major_faults = stats.major_faults,
vol_ctx = stats.vol_ctx,
invol_ctx = stats.invol_ctx,
in_blocks = stats.in_blocks,
out_blocks = stats.out_blocks,
"done"
);
if stats.swaps > 0 {