perf(query): replace static divisor with dynamic overhead multiplier
Replaces the static 16 divisor with a dynamic overhead_multiplier that scales chunk size based on n_genomes, the --detail flag, and a safety factor. This bounds per-chunk memory usage to ≤50% of available RAM across concurrent workers by accounting for genome-scaled k-mer buffers and optional coverage data.
This commit is contained in:
@@ -456,17 +456,49 @@ pub fn run(args: QueryArgs) {
|
||||
let n_workers = args.threads.max(1);
|
||||
|
||||
// Chunk size: each chunk stays in memory for its entire processing lifetime.
|
||||
// Overhead per raw byte is ~8× (Rope + parsed records + superkmers + results).
|
||||
// We target ≤ 50 % of available RAM across all concurrent workers.
|
||||
//
|
||||
// Per-chunk memory is dominated by two dense buffers that scale with
|
||||
// n_genomes — KmerResults.data and win_min (process_chunk) — each
|
||||
// roughly total_kmers_in_chunk × n_genomes × 4 bytes (u32), plus `cov`
|
||||
// (roughly) doubling that cost when --detail is set. total_kmers_in_chunk
|
||||
// ≈ chunk_bytes (one s-mer position per raw input byte; the per-sequence
|
||||
// k-1 tail is negligible at realistic chunk sizes).
|
||||
//
|
||||
// BASE_OVERHEAD approximates what scales with chunk_bytes alone,
|
||||
// independent of n_genomes: the Rope itself, parsed SeqRecord sequence +
|
||||
// normalised bytes, the superkmer dedup map, and the JSON output buffer.
|
||||
// Like the n_genomes-scaled term, this is an estimate — validate against
|
||||
// actual peak RSS (Stage::stop's `rss` in the summary table) on real
|
||||
// workloads rather than trusting it blindly.
|
||||
//
|
||||
// We target ≤ 50 % of available RAM across all concurrent workers
|
||||
// (SAFETY_FACTOR).
|
||||
const BASE_OVERHEAD: u64 = 4;
|
||||
const BYTES_PER_KMER_PER_GENOME: u64 = 8; // KmerResults.data + win_min, one u32 (4B) each
|
||||
const SAFETY_FACTOR: u64 = 2;
|
||||
|
||||
let detail_factor: u64 = if args.detail { 2 } else { 1 };
|
||||
let overhead_multiplier =
|
||||
BASE_OVERHEAD + n_genomes as u64 * BYTES_PER_KMER_PER_GENOME * detail_factor;
|
||||
|
||||
let chunk_bytes = args
|
||||
.chunk_size
|
||||
.map(|mb| mb * 1024 * 1024)
|
||||
.unwrap_or_else(|| {
|
||||
let avail = available_memory_bytes();
|
||||
let computed = avail / (n_workers as u64 * 16);
|
||||
let computed = avail / (n_workers as u64 * overhead_multiplier * SAFETY_FACTOR);
|
||||
computed.clamp(4 * 1024 * 1024, 256 * 1024 * 1024) as usize
|
||||
});
|
||||
|
||||
debug!(
|
||||
chunk_bytes,
|
||||
n_genomes,
|
||||
detail = args.detail,
|
||||
overhead_multiplier,
|
||||
estimated_peak_chunk_bytes = chunk_bytes as u64 * overhead_multiplier,
|
||||
"chunk-size formula resolved"
|
||||
);
|
||||
|
||||
let effective_z: usize = args
|
||||
.findere_z
|
||||
.unwrap_or_else(|| match idx.meta().config.evidence {
|
||||
|
||||
Reference in New Issue
Block a user