src/Cargo.lock

@@ -1704,7 +1704,7 @@ dependencies = [
 
 [[package]]
 name = "obikmer"
-version = "1.1.37"
+version = "1.1.38"
 dependencies = [
  "clap",
  "csv",

src/obikmer/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "obikmer"
-version = "1.1.37"
+version = "1.1.38"
 edition = "2024"
 
 [[bin]]

src/obikmer/src/cmd/query.rs

@@ -325,6 +325,42 @@ fn process_chunk(
     let batch = QueryBatch::from_records(records, k, 6, 0.7, n_partitions);
     let n_seqs = batch.ids.len();
 
+    // Estimate QueryBatch::by_partition's actual memory footprint: the
+    // k-mer-level dedup map (roadmap point 5) — one HashMap<CanonicalKmer,
+    // Vec<KmerDesc>> per partition, sized by *unique* k-mers, not shrunk by
+    // dedup. On real workloads with a low intra-chunk duplication rate this
+    // can dwarf every other per-chunk structure, including the sparse
+    // Findere ones logged further down — unlike those, chunk_bytes's formula
+    // (run()) does not account for this at all today. Measured by allocated
+    // capacity, not logical length, to reflect real memory pressure
+    // (HashMap/Vec growth slack) — `by_partition` is alive for the entire
+    // process_chunk call (never drained, only iterated by reference), so
+    // this is its footprint for the whole chunk lifetime, not a transient.
+    let hashmap_slot_bytes = (std::mem::size_of::<CanonicalKmer>()
+        + std::mem::size_of::<Vec<KmerDesc>>()
+        + 1) as u64; // +1 ≈ hashbrown control byte per slot
+    let by_partition_map_bytes: u64 = batch
+        .by_partition
+        .iter()
+        .map(|m| m.capacity() as u64 * hashmap_slot_bytes)
+        .sum();
+    let by_partition_desc_bytes: u64 = batch
+        .by_partition
+        .iter()
+        .flat_map(|m| m.values())
+        .map(|v| v.capacity() as u64 * std::mem::size_of::<KmerDesc>() as u64)
+        .sum();
+    let by_partition_bytes = by_partition_map_bytes + by_partition_desc_bytes;
+
+    debug!(
+        n_unique_kmers_total = batch.by_partition.iter().map(|m| m.len() as u64).sum::<u64>(),
+        by_partition_map_bytes,
+        by_partition_desc_bytes,
+        by_partition_bytes,
+        chunk_bytes,
+        "by_partition memory retained"
+    );
+
     // Sparse bookkeeping for the whole chunk:
     // - smer_index: O(total_smers) — is this s-mer in the index at all.
     // - by_genome[g]: raw (seq_idx, pos_smer, value) hits for genome g, only