perf(query): replace static divisor with dynamic overhead multiplier

Replaces the static 16 divisor with a dynamic overhead_multiplier that scales chunk size based on n_genomes, the --detail flag, and a safety factor. This bounds per-chunk memory usage to ≤50% of available RAM across concurrent workers by accounting for genome-scaled k-mer buffers and optional coverage data.
This commit is contained in:
Eric Coissac
2026-07-07 16:14:10 +02:00
parent 9d49929b0c
commit 9d7ced4493
+35 -3
View File
@@ -456,17 +456,49 @@ pub fn run(args: QueryArgs) {
let n_workers = args.threads.max(1); let n_workers = args.threads.max(1);
// Chunk size: each chunk stays in memory for its entire processing lifetime. // Chunk size: each chunk stays in memory for its entire processing lifetime.
// Overhead per raw byte is ~8× (Rope + parsed records + superkmers + results). //
// We target ≤ 50 % of available RAM across all concurrent workers. // Per-chunk memory is dominated by two dense buffers that scale with
// n_genomes — KmerResults.data and win_min (process_chunk) — each
// roughly total_kmers_in_chunk × n_genomes × 4 bytes (u32), plus `cov`
// (roughly) doubling that cost when --detail is set. total_kmers_in_chunk
// ≈ chunk_bytes (one s-mer position per raw input byte; the per-sequence
// k-1 tail is negligible at realistic chunk sizes).
//
// BASE_OVERHEAD approximates what scales with chunk_bytes alone,
// independent of n_genomes: the Rope itself, parsed SeqRecord sequence +
// normalised bytes, the superkmer dedup map, and the JSON output buffer.
// Like the n_genomes-scaled term, this is an estimate — validate against
// actual peak RSS (Stage::stop's `rss` in the summary table) on real
// workloads rather than trusting it blindly.
//
// We target ≤ 50 % of available RAM across all concurrent workers
// (SAFETY_FACTOR).
const BASE_OVERHEAD: u64 = 4;
const BYTES_PER_KMER_PER_GENOME: u64 = 8; // KmerResults.data + win_min, one u32 (4B) each
const SAFETY_FACTOR: u64 = 2;
let detail_factor: u64 = if args.detail { 2 } else { 1 };
let overhead_multiplier =
BASE_OVERHEAD + n_genomes as u64 * BYTES_PER_KMER_PER_GENOME * detail_factor;
let chunk_bytes = args let chunk_bytes = args
.chunk_size .chunk_size
.map(|mb| mb * 1024 * 1024) .map(|mb| mb * 1024 * 1024)
.unwrap_or_else(|| { .unwrap_or_else(|| {
let avail = available_memory_bytes(); let avail = available_memory_bytes();
let computed = avail / (n_workers as u64 * 16); let computed = avail / (n_workers as u64 * overhead_multiplier * SAFETY_FACTOR);
computed.clamp(4 * 1024 * 1024, 256 * 1024 * 1024) as usize computed.clamp(4 * 1024 * 1024, 256 * 1024 * 1024) as usize
}); });
debug!(
chunk_bytes,
n_genomes,
detail = args.detail,
overhead_multiplier,
estimated_peak_chunk_bytes = chunk_bytes as u64 * overhead_multiplier,
"chunk-size formula resolved"
);
let effective_z: usize = args let effective_z: usize = args
.findere_z .findere_z
.unwrap_or_else(|| match idx.meta().config.evidence { .unwrap_or_else(|| match idx.meta().config.evidence {