Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2740f52326 | ||
|
|
dff5d2f457 |
Generated
+1
-1
@@ -1704,7 +1704,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "obikmer"
|
name = "obikmer"
|
||||||
version = "1.1.37"
|
version = "1.1.38"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"clap",
|
"clap",
|
||||||
"csv",
|
"csv",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "obikmer"
|
name = "obikmer"
|
||||||
version = "1.1.37"
|
version = "1.1.38"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[[bin]]
|
[[bin]]
|
||||||
|
|||||||
@@ -325,6 +325,42 @@ fn process_chunk(
|
|||||||
let batch = QueryBatch::from_records(records, k, 6, 0.7, n_partitions);
|
let batch = QueryBatch::from_records(records, k, 6, 0.7, n_partitions);
|
||||||
let n_seqs = batch.ids.len();
|
let n_seqs = batch.ids.len();
|
||||||
|
|
||||||
|
// Estimate QueryBatch::by_partition's actual memory footprint: the
|
||||||
|
// k-mer-level dedup map (roadmap point 5) — one HashMap<CanonicalKmer,
|
||||||
|
// Vec<KmerDesc>> per partition, sized by *unique* k-mers, not shrunk by
|
||||||
|
// dedup. On real workloads with a low intra-chunk duplication rate this
|
||||||
|
// can dwarf every other per-chunk structure, including the sparse
|
||||||
|
// Findere ones logged further down — unlike those, chunk_bytes's formula
|
||||||
|
// (run()) does not account for this at all today. Measured by allocated
|
||||||
|
// capacity, not logical length, to reflect real memory pressure
|
||||||
|
// (HashMap/Vec growth slack) — `by_partition` is alive for the entire
|
||||||
|
// process_chunk call (never drained, only iterated by reference), so
|
||||||
|
// this is its footprint for the whole chunk lifetime, not a transient.
|
||||||
|
let hashmap_slot_bytes = (std::mem::size_of::<CanonicalKmer>()
|
||||||
|
+ std::mem::size_of::<Vec<KmerDesc>>()
|
||||||
|
+ 1) as u64; // +1 ≈ hashbrown control byte per slot
|
||||||
|
let by_partition_map_bytes: u64 = batch
|
||||||
|
.by_partition
|
||||||
|
.iter()
|
||||||
|
.map(|m| m.capacity() as u64 * hashmap_slot_bytes)
|
||||||
|
.sum();
|
||||||
|
let by_partition_desc_bytes: u64 = batch
|
||||||
|
.by_partition
|
||||||
|
.iter()
|
||||||
|
.flat_map(|m| m.values())
|
||||||
|
.map(|v| v.capacity() as u64 * std::mem::size_of::<KmerDesc>() as u64)
|
||||||
|
.sum();
|
||||||
|
let by_partition_bytes = by_partition_map_bytes + by_partition_desc_bytes;
|
||||||
|
|
||||||
|
debug!(
|
||||||
|
n_unique_kmers_total = batch.by_partition.iter().map(|m| m.len() as u64).sum::<u64>(),
|
||||||
|
by_partition_map_bytes,
|
||||||
|
by_partition_desc_bytes,
|
||||||
|
by_partition_bytes,
|
||||||
|
chunk_bytes,
|
||||||
|
"by_partition memory retained"
|
||||||
|
);
|
||||||
|
|
||||||
// Sparse bookkeeping for the whole chunk:
|
// Sparse bookkeeping for the whole chunk:
|
||||||
// - smer_index: O(total_smers) — is this s-mer in the index at all.
|
// - smer_index: O(total_smers) — is this s-mer in the index at all.
|
||||||
// - by_genome[g]: raw (seq_idx, pos_smer, value) hits for genome g, only
|
// - by_genome[g]: raw (seq_idx, pos_smer, value) hits for genome g, only
|
||||||
|
|||||||
Reference in New Issue
Block a user