perf(query): replace static divisor with dynamic overhead multiplier
Replaces the static 16 divisor with a dynamic overhead_multiplier that scales chunk size based on n_genomes, the --detail flag, and a safety factor. This bounds per-chunk memory usage to ≤50% of available RAM across concurrent workers by accounting for genome-scaled k-mer buffers and optional coverage data.
This commit is contained in:
@@ -456,17 +456,49 @@ pub fn run(args: QueryArgs) {
|
|||||||
let n_workers = args.threads.max(1);
|
let n_workers = args.threads.max(1);
|
||||||
|
|
||||||
// Chunk size: each chunk stays in memory for its entire processing lifetime.
|
// Chunk size: each chunk stays in memory for its entire processing lifetime.
|
||||||
// Overhead per raw byte is ~8× (Rope + parsed records + superkmers + results).
|
//
|
||||||
// We target ≤ 50 % of available RAM across all concurrent workers.
|
// Per-chunk memory is dominated by two dense buffers that scale with
|
||||||
|
// n_genomes — KmerResults.data and win_min (process_chunk) — each
|
||||||
|
// roughly total_kmers_in_chunk × n_genomes × 4 bytes (u32), plus `cov`
|
||||||
|
// (roughly) doubling that cost when --detail is set. total_kmers_in_chunk
|
||||||
|
// ≈ chunk_bytes (one s-mer position per raw input byte; the per-sequence
|
||||||
|
// k-1 tail is negligible at realistic chunk sizes).
|
||||||
|
//
|
||||||
|
// BASE_OVERHEAD approximates what scales with chunk_bytes alone,
|
||||||
|
// independent of n_genomes: the Rope itself, parsed SeqRecord sequence +
|
||||||
|
// normalised bytes, the superkmer dedup map, and the JSON output buffer.
|
||||||
|
// Like the n_genomes-scaled term, this is an estimate — validate against
|
||||||
|
// actual peak RSS (Stage::stop's `rss` in the summary table) on real
|
||||||
|
// workloads rather than trusting it blindly.
|
||||||
|
//
|
||||||
|
// We target ≤ 50 % of available RAM across all concurrent workers
|
||||||
|
// (SAFETY_FACTOR).
|
||||||
|
const BASE_OVERHEAD: u64 = 4;
|
||||||
|
const BYTES_PER_KMER_PER_GENOME: u64 = 8; // KmerResults.data + win_min, one u32 (4B) each
|
||||||
|
const SAFETY_FACTOR: u64 = 2;
|
||||||
|
|
||||||
|
let detail_factor: u64 = if args.detail { 2 } else { 1 };
|
||||||
|
let overhead_multiplier =
|
||||||
|
BASE_OVERHEAD + n_genomes as u64 * BYTES_PER_KMER_PER_GENOME * detail_factor;
|
||||||
|
|
||||||
let chunk_bytes = args
|
let chunk_bytes = args
|
||||||
.chunk_size
|
.chunk_size
|
||||||
.map(|mb| mb * 1024 * 1024)
|
.map(|mb| mb * 1024 * 1024)
|
||||||
.unwrap_or_else(|| {
|
.unwrap_or_else(|| {
|
||||||
let avail = available_memory_bytes();
|
let avail = available_memory_bytes();
|
||||||
let computed = avail / (n_workers as u64 * 16);
|
let computed = avail / (n_workers as u64 * overhead_multiplier * SAFETY_FACTOR);
|
||||||
computed.clamp(4 * 1024 * 1024, 256 * 1024 * 1024) as usize
|
computed.clamp(4 * 1024 * 1024, 256 * 1024 * 1024) as usize
|
||||||
});
|
});
|
||||||
|
|
||||||
|
debug!(
|
||||||
|
chunk_bytes,
|
||||||
|
n_genomes,
|
||||||
|
detail = args.detail,
|
||||||
|
overhead_multiplier,
|
||||||
|
estimated_peak_chunk_bytes = chunk_bytes as u64 * overhead_multiplier,
|
||||||
|
"chunk-size formula resolved"
|
||||||
|
);
|
||||||
|
|
||||||
let effective_z: usize = args
|
let effective_z: usize = args
|
||||||
.findere_z
|
.findere_z
|
||||||
.unwrap_or_else(|| match idx.meta().config.evidence {
|
.unwrap_or_else(|| match idx.meta().config.evidence {
|
||||||
|
|||||||
Reference in New Issue
Block a user