Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2740f52326 | ||
|
|
dff5d2f457 |
Generated
+1
-1
@@ -1704,7 +1704,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "obikmer"
|
||||
version = "1.1.37"
|
||||
version = "1.1.38"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"csv",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "obikmer"
|
||||
version = "1.1.37"
|
||||
version = "1.1.38"
|
||||
edition = "2024"
|
||||
|
||||
[[bin]]
|
||||
|
||||
@@ -325,6 +325,42 @@ fn process_chunk(
|
||||
let batch = QueryBatch::from_records(records, k, 6, 0.7, n_partitions);
|
||||
let n_seqs = batch.ids.len();
|
||||
|
||||
// Estimate QueryBatch::by_partition's actual memory footprint: the
|
||||
// k-mer-level dedup map (roadmap point 5) — one HashMap<CanonicalKmer,
|
||||
// Vec<KmerDesc>> per partition, sized by *unique* k-mers, not shrunk by
|
||||
// dedup. On real workloads with a low intra-chunk duplication rate this
|
||||
// can dwarf every other per-chunk structure, including the sparse
|
||||
// Findere ones logged further down — unlike those, chunk_bytes's formula
|
||||
// (run()) does not account for this at all today. Measured by allocated
|
||||
// capacity, not logical length, to reflect real memory pressure
|
||||
// (HashMap/Vec growth slack) — `by_partition` is alive for the entire
|
||||
// process_chunk call (never drained, only iterated by reference), so
|
||||
// this is its footprint for the whole chunk lifetime, not a transient.
|
||||
let hashmap_slot_bytes = (std::mem::size_of::<CanonicalKmer>()
|
||||
+ std::mem::size_of::<Vec<KmerDesc>>()
|
||||
+ 1) as u64; // +1 ≈ hashbrown control byte per slot
|
||||
let by_partition_map_bytes: u64 = batch
|
||||
.by_partition
|
||||
.iter()
|
||||
.map(|m| m.capacity() as u64 * hashmap_slot_bytes)
|
||||
.sum();
|
||||
let by_partition_desc_bytes: u64 = batch
|
||||
.by_partition
|
||||
.iter()
|
||||
.flat_map(|m| m.values())
|
||||
.map(|v| v.capacity() as u64 * std::mem::size_of::<KmerDesc>() as u64)
|
||||
.sum();
|
||||
let by_partition_bytes = by_partition_map_bytes + by_partition_desc_bytes;
|
||||
|
||||
debug!(
|
||||
n_unique_kmers_total = batch.by_partition.iter().map(|m| m.len() as u64).sum::<u64>(),
|
||||
by_partition_map_bytes,
|
||||
by_partition_desc_bytes,
|
||||
by_partition_bytes,
|
||||
chunk_bytes,
|
||||
"by_partition memory retained"
|
||||
);
|
||||
|
||||
// Sparse bookkeeping for the whole chunk:
|
||||
// - smer_index: O(total_smers) — is this s-mer in the index at all.
|
||||
// - by_genome[g]: raw (seq_idx, pos_smer, value) hits for genome g, only
|
||||
|
||||
Reference in New Issue
Block a user