diff --git a/src/Cargo.lock b/src/Cargo.lock index 28e2eea..3e31a1e 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1704,7 +1704,7 @@ dependencies = [ [[package]] name = "obikmer" -version = "1.1.37" +version = "1.1.38" dependencies = [ "clap", "csv", diff --git a/src/obikmer/Cargo.toml b/src/obikmer/Cargo.toml index ef80137..d9ea7a6 100644 --- a/src/obikmer/Cargo.toml +++ b/src/obikmer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "obikmer" -version = "1.1.37" +version = "1.1.38" edition = "2024" [[bin]] diff --git a/src/obikmer/src/cmd/query.rs b/src/obikmer/src/cmd/query.rs index a5ab5e7..900aeb7 100644 --- a/src/obikmer/src/cmd/query.rs +++ b/src/obikmer/src/cmd/query.rs @@ -325,6 +325,42 @@ fn process_chunk( let batch = QueryBatch::from_records(records, k, 6, 0.7, n_partitions); let n_seqs = batch.ids.len(); + // Estimate QueryBatch::by_partition's actual memory footprint: the + // k-mer-level dedup map (roadmap point 5) — one HashMap> per partition, sized by *unique* k-mers, not shrunk by + // dedup. On real workloads with a low intra-chunk duplication rate this + // can dwarf every other per-chunk structure, including the sparse + // Findere ones logged further down — unlike those, chunk_bytes's formula + // (run()) does not account for this at all today. Measured by allocated + // capacity, not logical length, to reflect real memory pressure + // (HashMap/Vec growth slack) — `by_partition` is alive for the entire + // process_chunk call (never drained, only iterated by reference), so + // this is its footprint for the whole chunk lifetime, not a transient. + let hashmap_slot_bytes = (std::mem::size_of::() + + std::mem::size_of::>() + + 1) as u64; // +1 ≈ hashbrown control byte per slot + let by_partition_map_bytes: u64 = batch + .by_partition + .iter() + .map(|m| m.capacity() as u64 * hashmap_slot_bytes) + .sum(); + let by_partition_desc_bytes: u64 = batch + .by_partition + .iter() + .flat_map(|m| m.values()) + .map(|v| v.capacity() as u64 * std::mem::size_of::() as u64) + .sum(); + let by_partition_bytes = by_partition_map_bytes + by_partition_desc_bytes; + + debug!( + n_unique_kmers_total = batch.by_partition.iter().map(|m| m.len() as u64).sum::(), + by_partition_map_bytes, + by_partition_desc_bytes, + by_partition_bytes, + chunk_bytes, + "by_partition memory retained" + ); + // Sparse bookkeeping for the whole chunk: // - smer_index: O(total_smers) — is this s-mer in the index at all. // - by_genome[g]: raw (seq_idx, pos_smer, value) hits for genome g, only