feat: add pack optimization prompt, debug infrastructure, and monitoring scripts
Add a prompt template for optimizing the `pack` step and a suite of Docker-based debug tools including build, run, and monitoring scripts. Enhance `pack_matrices` with LFD partition sizing and detailed timing reports, and update `stop` with additional system statistics. - Add `prompts/pack_optimization.md` to guide optimization of the `pack` step, specifying context, performance measurements, and deliverable structures. - Add `scripts/docker_debug/` infrastructure: Dockerfile for musl targets, `build.sh` for image creation, `run.sh` for parallel indexing and merging, `monitor.sh` for PID tracking, `gen_fasta.py` for synthetic genomes, and `gen_fasta_real.py` for real genome generation with `read_fna`. - Add `scripts/monitor_pack.sh` as a 1 Hz sampler for I/O and memory metrics, writing samples to CSV and summary reports. - Add `scripts/watch_merge.sh` to monitor process resource usage, enforcing memory limits via SIGTERM/SIGKILL guards based on VmRSS and VmSize. - Modify `src/obikindex/src/index/kmer_index.rs`: add `pending_matrix_bytes` and `report_pack_timings`; update `pack_matrices` to use LFD strategy for partition sizing and clarify documentation regarding `stat()` proxy bytes. - Modify `src/obisys/src/stage.rs`: update `stop` definition to include new variables (`user_secs`, `sys_secs`, faults, context switches, I/O blocks) and logic checking for swap usage.
This commit is contained in:
1 parent
24291a4484
commit
03eaa4544a
11 files changed
+497
-2
No files matched your search
@@ -296,7 +296,14 @@ impl KmerIndex {
|
||||
};
|
||||
|
||||
let n = self.n_partitions();
|
||||
let order: Vec<usize> = (0..n).collect();
|
||||
// Largest partitions first (LFD), so the long jobs do not end up as
|
||||
// stragglers at the tail. Sizes are `stat()` only; the sort is stable,
|
||||
// ties keep partition-number order. Partitions are independent, so
|
||||
// the order has no effect on what is written.
|
||||
let sizes: Vec<u64> = (0..n).map(|i| self.pending_matrix_bytes(i)).collect();
|
||||
let mut order: Vec<usize> = (0..n).collect();
|
||||
order.sort_by_key(|&i| std::cmp::Reverse(sizes[i]));
|
||||
let mut timings: Vec<(usize, u64, std::time::Duration)> = Vec::with_capacity(n);
|
||||
let pb = progress_bar("pack", n as u64, "partitions");
|
||||
obisys::PartitionRunner::new().run(
|
||||
&order,
|
||||
@@ -327,14 +334,50 @@ impl KmerIndex {
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
|_, _, _| {
|
||||
|i, _, elapsed| {
|
||||
tracing::debug!(
|
||||
partition = i,
|
||||
matrix_bytes = sizes[i],
|
||||
secs = elapsed.as_secs_f64(),
|
||||
"pack: partition done"
|
||||
);
|
||||
timings.push((i, sizes[i], elapsed));
|
||||
pb.inc(1);
|
||||
},
|
||||
)?;
|
||||
pb.finish_and_clear();
|
||||
report_pack_timings(&timings);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Bytes of matrix data (`presence/` and `counts/` of every layer) still
|
||||
/// present in partition `i`, from `stat()` only. Before packing this is
|
||||
/// the columnar data, i.e. a proxy for the work `pack_matrices` has to do
|
||||
/// on that partition. Missing partition or directories count as 0.
|
||||
pub fn pending_matrix_bytes(&self, i: usize) -> u64 {
|
||||
let Ok(n_layers) = self.n_layers(i) else {
|
||||
return 0;
|
||||
};
|
||||
let mut total = 0u64;
|
||||
for l in 0..n_layers {
|
||||
let Ok(layer_dir) = self.layer_dir(i, l) else {
|
||||
continue;
|
||||
};
|
||||
for sub in ["presence", "counts"] {
|
||||
let Ok(entries) = fs::read_dir(layer_dir.join(sub)) else {
|
||||
continue;
|
||||
};
|
||||
total += entries
|
||||
.flatten()
|
||||
.filter_map(|e| e.metadata().ok())
|
||||
.filter(|m| m.is_file())
|
||||
.map(|m| m.len())
|
||||
.sum::<u64>();
|
||||
}
|
||||
}
|
||||
total
|
||||
}
|
||||
|
||||
// /// Write a `layer_meta.json` in any layer directory that is missing one.
|
||||
// ///
|
||||
// /// Old indexes were built before this file was required. The number of
|
||||
@@ -394,3 +437,34 @@ impl KmerIndex {
|
||||
// Ok(())
|
||||
// }
|
||||
}
|
||||
|
||||
/// One `info!` summary of `pack_matrices` per-partition timings: the sum of
|
||||
/// partition times (divide by the stage wall time to get the mean number of
|
||||
/// concurrently busy workers), the median and maximum, and the slowest
|
||||
/// partitions with their pre-pack matrix size. `timings` holds
|
||||
/// `(partition, matrix_bytes, elapsed)`.
|
||||
fn report_pack_timings(timings: &[(usize, u64, std::time::Duration)]) {
|
||||
if timings.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut by_time: Vec<&(usize, u64, std::time::Duration)> = timings.iter().collect();
|
||||
by_time.sort_by_key(|t| std::cmp::Reverse(t.2));
|
||||
let sum: f64 = timings.iter().map(|t| t.2.as_secs_f64()).sum();
|
||||
let bytes: u64 = timings.iter().map(|t| t.1).sum();
|
||||
let median = by_time[by_time.len() / 2].2.as_secs_f64();
|
||||
tracing::info!(
|
||||
"pack: {} partition(s), sum of partition times {:.0}s, median {:.1}s, max {:.1}s, {:.1} GB of matrix data",
|
||||
timings.len(),
|
||||
sum,
|
||||
median,
|
||||
by_time[0].2.as_secs_f64(),
|
||||
bytes as f64 / 1e9,
|
||||
);
|
||||
for (i, b, d) in by_time.iter().take(5).map(|t| (t.0, t.1, t.2)) {
|
||||
tracing::info!(
|
||||
"pack: slowest partition {i}: {:.1}s, {:.2} GB of matrix data",
|
||||
d.as_secs_f64(),
|
||||
b as f64 / 1e9,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,14 @@ impl Stage {
|
||||
wall_secs = format_args!("{:.3}", stats.wall_secs),
|
||||
rss = %fmt_bytes(stats.max_rss_bytes),
|
||||
swaps = stats.swaps,
|
||||
user_secs = format_args!("{:.1}", stats.user_secs),
|
||||
sys_secs = format_args!("{:.1}", stats.sys_secs),
|
||||
minor_faults = stats.minor_faults,
|
||||
major_faults = stats.major_faults,
|
||||
vol_ctx = stats.vol_ctx,
|
||||
invol_ctx = stats.invol_ctx,
|
||||
in_blocks = stats.in_blocks,
|
||||
out_blocks = stats.out_blocks,
|
||||
"done"
|
||||
);
|
||||
if stats.swaps > 0 {
|
||||
|
||||
Reference in new issue
Block a user