diff --git a/src/obikmer/src/cmd/query.rs b/src/obikmer/src/cmd/query.rs index b2a2af6..ab03b87 100644 --- a/src/obikmer/src/cmd/query.rs +++ b/src/obikmer/src/cmd/query.rs @@ -456,17 +456,49 @@ pub fn run(args: QueryArgs) { let n_workers = args.threads.max(1); // Chunk size: each chunk stays in memory for its entire processing lifetime. - // Overhead per raw byte is ~8× (Rope + parsed records + superkmers + results). - // We target ≤ 50 % of available RAM across all concurrent workers. + // + // Per-chunk memory is dominated by two dense buffers that scale with + // n_genomes — KmerResults.data and win_min (process_chunk) — each + // roughly total_kmers_in_chunk × n_genomes × 4 bytes (u32), plus `cov` + // (roughly) doubling that cost when --detail is set. total_kmers_in_chunk + // ≈ chunk_bytes (one s-mer position per raw input byte; the per-sequence + // k-1 tail is negligible at realistic chunk sizes). + // + // BASE_OVERHEAD approximates what scales with chunk_bytes alone, + // independent of n_genomes: the Rope itself, parsed SeqRecord sequence + + // normalised bytes, the superkmer dedup map, and the JSON output buffer. + // Like the n_genomes-scaled term, this is an estimate — validate against + // actual peak RSS (Stage::stop's `rss` in the summary table) on real + // workloads rather than trusting it blindly. + // + // We target ≤ 50 % of available RAM across all concurrent workers + // (SAFETY_FACTOR). + const BASE_OVERHEAD: u64 = 4; + const BYTES_PER_KMER_PER_GENOME: u64 = 8; // KmerResults.data + win_min, one u32 (4B) each + const SAFETY_FACTOR: u64 = 2; + + let detail_factor: u64 = if args.detail { 2 } else { 1 }; + let overhead_multiplier = + BASE_OVERHEAD + n_genomes as u64 * BYTES_PER_KMER_PER_GENOME * detail_factor; + let chunk_bytes = args .chunk_size .map(|mb| mb * 1024 * 1024) .unwrap_or_else(|| { let avail = available_memory_bytes(); - let computed = avail / (n_workers as u64 * 16); + let computed = avail / (n_workers as u64 * overhead_multiplier * SAFETY_FACTOR); computed.clamp(4 * 1024 * 1024, 256 * 1024 * 1024) as usize }); + debug!( + chunk_bytes, + n_genomes, + detail = args.detail, + overhead_multiplier, + estimated_peak_chunk_bytes = chunk_bytes as u64 * overhead_multiplier, + "chunk-size formula resolved" + ); + let effective_z: usize = args .findere_z .unwrap_or_else(|| match idx.meta().config.evidence {