src/obikmer/src/cmd/query.rs

@@ -456,17 +456,49 @@ pub fn run(args: QueryArgs) {
     let n_workers = args.threads.max(1);
 
     // Chunk size: each chunk stays in memory for its entire processing lifetime.
-    // Overhead per raw byte is ~8× (Rope + parsed records + superkmers + results).
-    // We target ≤ 50 % of available RAM across all concurrent workers.
+    //
+    // Per-chunk memory is dominated by two dense buffers that scale with
+    // n_genomes — KmerResults.data and win_min (process_chunk) — each
+    // roughly total_kmers_in_chunk × n_genomes × 4 bytes (u32), plus `cov`
+    // (roughly) doubling that cost when --detail is set. total_kmers_in_chunk
+    // ≈ chunk_bytes (one s-mer position per raw input byte; the per-sequence
+    // k-1 tail is negligible at realistic chunk sizes).
+    //
+    // BASE_OVERHEAD approximates what scales with chunk_bytes alone,
+    // independent of n_genomes: the Rope itself, parsed SeqRecord sequence +
+    // normalised bytes, the superkmer dedup map, and the JSON output buffer.
+    // Like the n_genomes-scaled term, this is an estimate — validate against
+    // actual peak RSS (Stage::stop's `rss` in the summary table) on real
+    // workloads rather than trusting it blindly.
+    //
+    // We target ≤ 50 % of available RAM across all concurrent workers
+    // (SAFETY_FACTOR).
+    const BASE_OVERHEAD: u64 = 4;
+    const BYTES_PER_KMER_PER_GENOME: u64 = 8; // KmerResults.data + win_min, one u32 (4B) each
+    const SAFETY_FACTOR: u64 = 2;
+
+    let detail_factor: u64 = if args.detail { 2 } else { 1 };
+    let overhead_multiplier =
+        BASE_OVERHEAD + n_genomes as u64 * BYTES_PER_KMER_PER_GENOME * detail_factor;
+
     let chunk_bytes = args
         .chunk_size
         .map(|mb| mb * 1024 * 1024)
         .unwrap_or_else(|| {
             let avail = available_memory_bytes();
-            let computed = avail / (n_workers as u64 * 16);
+            let computed = avail / (n_workers as u64 * overhead_multiplier * SAFETY_FACTOR);
             computed.clamp(4 * 1024 * 1024, 256 * 1024 * 1024) as usize
         });
 
+    debug!(
+        chunk_bytes,
+        n_genomes,
+        detail = args.detail,
+        overhead_multiplier,
+        estimated_peak_chunk_bytes = chunk_bytes as u64 * overhead_multiplier,
+        "chunk-size formula resolved"
+    );
+
     let effective_z: usize = args
         .findere_z
         .unwrap_or_else(|| match idx.meta().config.evidence {