Refactor k-mer storage to use flat arrays
This change replaces the HashMap-based k-mer storage mechanism with a compact, flat array structure for `PartitionKmers`. This significantly reduces memory usage per partition and improves query execution time. It also updates the batch processing logic to handle the new data structure, introduces detailed memory footprint calculations, and updates relevant query layer APIs. Test coverage has been expanded to validate correct partitioning and k-mer ordering.
This commit is contained in:
1 parent
fe38424682
commit
09c80468b2
8 files changed
+156
-56
No files matched your search
+17
-11
@@ -1,13 +1,12 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use obikseq::CanonicalKmer;
|
||||
use obiread::record::SeqRecord;
|
||||
use obiskbuilder::SuperKmerIter;
|
||||
|
||||
use crate::partition_kmers::PartitionKmers;
|
||||
use crate::query_layer::KmerDesc;
|
||||
|
||||
/// A batch of query sequences, with k-mers deduplicated directly (not just at
|
||||
/// the superkmer level) and pre-split by partition.
|
||||
/// A batch of query sequences, with k-mers grouped by canonical k-mer (not just
|
||||
/// at the superkmer level) and pre-split by partition.
|
||||
///
|
||||
/// Superkmer *construction* (`SuperKmerIter`) is still required — it's the
|
||||
/// mechanism that computes minimizers and partition routing — but the dedup
|
||||
@@ -23,8 +22,8 @@ pub(crate) struct QueryBatch {
|
||||
pub(crate) seqs: Vec<Vec<u8>>,
|
||||
/// Total kmer count per sequence (used for `--detail` coverage allocation).
|
||||
pub(crate) n_kmers: Vec<u32>,
|
||||
/// Deduplicated k-mer occurrences, one map per partition.
|
||||
pub(crate) by_partition: Vec<HashMap<CanonicalKmer, Vec<KmerDesc>>>,
|
||||
/// K-mer occurrences grouped by k-mer, one entry per partition.
|
||||
pub(crate) by_partition: Vec<PartitionKmers>,
|
||||
}
|
||||
|
||||
impl QueryBatch {
|
||||
@@ -41,20 +40,20 @@ impl QueryBatch {
|
||||
let mut seqs = Vec::with_capacity(records.len());
|
||||
let mut n_kmers = Vec::with_capacity(records.len());
|
||||
let mask = (n_partitions as u64) - 1;
|
||||
let mut by_partition: Vec<HashMap<CanonicalKmer, Vec<KmerDesc>>> =
|
||||
(0..n_partitions).map(|_| HashMap::new()).collect();
|
||||
let mut occurrences: Vec<Vec<(CanonicalKmer, KmerDesc)>> =
|
||||
(0..n_partitions).map(|_| Vec::new()).collect();
|
||||
|
||||
for (seq_idx, record) in records.into_iter().enumerate() {
|
||||
let mut kmer_offset = 0u32;
|
||||
|
||||
for rsk in SuperKmerIter::new(&record.normalized, k, level_max, theta) {
|
||||
let part_idx = (rsk.minimizer().seq_hash() & mask) as usize;
|
||||
let map = &mut by_partition[part_idx];
|
||||
let part = &mut occurrences[part_idx];
|
||||
for (j, kmer) in rsk.superkmer().iter_canonical_kmers().enumerate() {
|
||||
map.entry(kmer).or_default().push(KmerDesc {
|
||||
part.push((kmer, KmerDesc {
|
||||
seq_idx: seq_idx as u32,
|
||||
pos: kmer_offset + j as u32,
|
||||
});
|
||||
}));
|
||||
}
|
||||
let n = (rsk.seql() - k + 1) as u32;
|
||||
kmer_offset += n;
|
||||
@@ -65,6 +64,13 @@ impl QueryBatch {
|
||||
n_kmers.push(kmer_offset);
|
||||
}
|
||||
|
||||
// One partition at a time: each 16-byte-per-occurrence buffer is freed
|
||||
// as soon as its compact form exists.
|
||||
let by_partition = occurrences
|
||||
.into_iter()
|
||||
.map(PartitionKmers::from_occurrences)
|
||||
.collect();
|
||||
|
||||
Self {
|
||||
ids,
|
||||
seqs,
|
||||
|
||||
@@ -3,7 +3,6 @@ use std::time::Instant;
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::GenomeInfo;
|
||||
use obikrope::Rope;
|
||||
use obikseq::CanonicalKmer;
|
||||
use obiread::record::parse_chunk;
|
||||
use tracing::debug;
|
||||
|
||||
@@ -62,37 +61,13 @@ pub fn process_chunk(
|
||||
let batch = QueryBatch::from_records(records, k, 6, 0.7, n_partitions);
|
||||
let n_seqs = batch.ids.len();
|
||||
|
||||
// Estimate QueryBatch::by_partition's actual memory footprint: the
|
||||
// k-mer-level dedup map — one HashMap<CanonicalKmer, Vec<KmerDesc>> per
|
||||
// partition, sized by *unique* k-mers, not shrunk by dedup. On real
|
||||
// workloads with a low intra-chunk duplication rate this can dwarf every
|
||||
// other per-chunk structure, including the sparse Findere ones logged
|
||||
// further down. Measured by allocated capacity, not logical length, to
|
||||
// reflect real memory pressure (HashMap/Vec growth slack) — `by_partition`
|
||||
// is alive for the entire process_chunk call (never drained, only
|
||||
// iterated by reference), so this is its footprint for the whole chunk
|
||||
// lifetime, not a transient.
|
||||
let hashmap_slot_bytes = (std::mem::size_of::<CanonicalKmer>()
|
||||
+ std::mem::size_of::<Vec<crate::query_layer::KmerDesc>>()
|
||||
+ 1) as u64; // +1 ≈ hashbrown control byte per slot
|
||||
let by_partition_map_bytes: u64 = batch
|
||||
.by_partition
|
||||
.iter()
|
||||
.map(|m| m.capacity() as u64 * hashmap_slot_bytes)
|
||||
.sum();
|
||||
let by_partition_desc_bytes: u64 = batch
|
||||
.by_partition
|
||||
.iter()
|
||||
.flat_map(|m| m.values())
|
||||
.map(|v| v.capacity() as u64 * std::mem::size_of::<crate::query_layer::KmerDesc>() as u64)
|
||||
.sum();
|
||||
let by_partition_bytes = by_partition_map_bytes + by_partition_desc_bytes;
|
||||
|
||||
// `by_partition` is alive for the entire process_chunk call (never
|
||||
// drained, only iterated by reference): this is its footprint for the
|
||||
// whole chunk lifetime. Measured by allocated capacity, not logical length.
|
||||
debug!(
|
||||
n_unique_kmers_total = batch.by_partition.iter().map(|m| m.len() as u64).sum::<u64>(),
|
||||
by_partition_map_bytes,
|
||||
by_partition_desc_bytes,
|
||||
by_partition_bytes,
|
||||
n_unique_kmers_total = batch.by_partition.iter().map(|p| p.len() as u64).sum::<u64>(),
|
||||
n_occurrences_total = batch.by_partition.iter().map(|p| p.n_occurrences() as u64).sum::<u64>(),
|
||||
by_partition_bytes = batch.by_partition.iter().map(|p| p.allocated_bytes() as u64).sum::<u64>(),
|
||||
chunk_bytes,
|
||||
"by_partition memory retained"
|
||||
);
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
//! the data model, never the reverse), same pattern as `obikindexer` for the
|
||||
//! build side.
|
||||
|
||||
mod partition_kmers;
|
||||
mod query_layer;
|
||||
mod batch;
|
||||
mod chunk;
|
||||
@@ -12,5 +13,6 @@ mod findere;
|
||||
mod smer_index;
|
||||
mod output;
|
||||
|
||||
pub use partition_kmers::PartitionKmers;
|
||||
pub use query_layer::{KmerDesc, QueryHit, QueryPartition, QueryStats};
|
||||
pub use chunk::process_chunk;
|
||||
@@ -0,0 +1,82 @@
|
||||
//! The k-mers of one query chunk routed to one partition, grouped by k-mer.
|
||||
//!
|
||||
//! Replaces a `HashMap<CanonicalKmer, Vec<KmerDesc>>`: with genomic reads
|
||||
//! almost every k-mer occurs once, so the map paid a table slot, a growth
|
||||
//! slack and a separate heap `Vec` for each unique k-mer (~70 bytes measured)
|
||||
//! for a dedup that removes under 10 % of the occurrences. Here the
|
||||
//! occurrences are sorted by k-mer and stored as three flat arrays — one
|
||||
//! 8-byte k-mer and one 4-byte offset per unique k-mer, one 8-byte
|
||||
//! [`KmerDesc`] per occurrence — about 20 bytes per unique k-mer, no
|
||||
//! per-k-mer allocation.
|
||||
|
||||
use obikseq::CanonicalKmer;
|
||||
|
||||
use crate::query_layer::KmerDesc;
|
||||
|
||||
/// Unique k-mers of one partition (sorted by raw value) and, for each, the
|
||||
/// slice of its occurrences (sorted by `(seq_idx, pos)`, i.e. query order).
|
||||
pub struct PartitionKmers {
|
||||
kmers: Vec<CanonicalKmer>,
|
||||
/// `descs[starts[i]..starts[i + 1]]` are the occurrences of `kmers[i]`.
|
||||
starts: Vec<u32>,
|
||||
descs: Vec<KmerDesc>,
|
||||
}
|
||||
|
||||
impl PartitionKmers {
|
||||
pub fn empty() -> Self {
|
||||
Self { kmers: Vec::new(), starts: vec![0], descs: Vec::new() }
|
||||
}
|
||||
|
||||
/// Group `occurrences` by k-mer. Consumes the (16 bytes per occurrence)
|
||||
/// input; after the call only the compact arrays remain.
|
||||
pub fn from_occurrences(mut occurrences: Vec<(CanonicalKmer, KmerDesc)>) -> Self {
|
||||
occurrences.sort_unstable_by_key(|(kmer, d)| (kmer.raw(), d.seq_idx, d.pos));
|
||||
|
||||
let mut kmers = Vec::new();
|
||||
let mut starts = Vec::new();
|
||||
let mut descs = Vec::with_capacity(occurrences.len());
|
||||
for (i, (kmer, desc)) in occurrences.iter().enumerate() {
|
||||
if kmers.last() != Some(kmer) {
|
||||
kmers.push(*kmer);
|
||||
starts.push(offset(i));
|
||||
}
|
||||
descs.push(*desc);
|
||||
}
|
||||
starts.push(offset(descs.len()));
|
||||
kmers.shrink_to_fit();
|
||||
starts.shrink_to_fit();
|
||||
Self { kmers, starts, descs }
|
||||
}
|
||||
|
||||
/// Number of unique k-mers.
|
||||
pub fn len(&self) -> usize {
|
||||
self.kmers.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.kmers.is_empty()
|
||||
}
|
||||
|
||||
/// Total number of occurrences (before grouping).
|
||||
pub fn n_occurrences(&self) -> usize {
|
||||
self.descs.len()
|
||||
}
|
||||
|
||||
/// `(kmer, its occurrences)` for every unique k-mer, in k-mer order.
|
||||
pub fn iter(&self) -> impl Iterator<Item = (CanonicalKmer, &[KmerDesc])> + '_ {
|
||||
self.kmers.iter().enumerate().map(|(i, &kmer)| {
|
||||
(kmer, &self.descs[self.starts[i] as usize..self.starts[i + 1] as usize])
|
||||
})
|
||||
}
|
||||
|
||||
/// Heap bytes held (by allocated capacity).
|
||||
pub fn allocated_bytes(&self) -> usize {
|
||||
self.kmers.capacity() * std::mem::size_of::<CanonicalKmer>()
|
||||
+ self.starts.capacity() * std::mem::size_of::<u32>()
|
||||
+ self.descs.capacity() * std::mem::size_of::<KmerDesc>()
|
||||
}
|
||||
}
|
||||
|
||||
fn offset(i: usize) -> u32 {
|
||||
u32::try_from(i).expect("more than 2^32 k-mer occurrences in one partition of one chunk — lower --chunk-size")
|
||||
}
|
||||
@@ -3,6 +3,8 @@ use std::collections::HashMap;
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikseq::CanonicalKmer;
|
||||
|
||||
use crate::partition_kmers::PartitionKmers;
|
||||
|
||||
// ── KmerDesc — one occurrence of a k-mer in the query batch ──────────────────
|
||||
|
||||
/// Describes one occurrence of a (deduplicated) k-mer in the query batch:
|
||||
@@ -100,7 +102,7 @@ pub trait QueryPartition {
|
||||
fn query_partition_with<F>(
|
||||
&self,
|
||||
part_idx: usize,
|
||||
kmers: &HashMap<CanonicalKmer, Vec<KmerDesc>>,
|
||||
kmers: &PartitionKmers,
|
||||
n_genomes: usize,
|
||||
on_event: F,
|
||||
) -> QueryStats
|
||||
@@ -112,7 +114,7 @@ impl QueryPartition for IndexCache {
|
||||
fn query_partition_with<F>(
|
||||
&self,
|
||||
part_idx: usize,
|
||||
kmers: &HashMap<CanonicalKmer, Vec<KmerDesc>>,
|
||||
kmers: &PartitionKmers,
|
||||
n_genomes: usize,
|
||||
mut on_event: F,
|
||||
) -> QueryStats
|
||||
@@ -131,7 +133,7 @@ impl QueryPartition for IndexCache {
|
||||
};
|
||||
|
||||
// ── Stage 1: MPHF-only pass, bucket hits by (layer_idx, slot) ────────
|
||||
let mut by_layer: Vec<HashMap<usize, &Vec<KmerDesc>>> =
|
||||
let mut by_layer: Vec<HashMap<usize, &[KmerDesc]>> =
|
||||
(0..n_layer).map(|_| HashMap::new()).collect();
|
||||
|
||||
// Layer-major over fixed-size chunks of the (unordered) kmer map:
|
||||
@@ -141,12 +143,12 @@ impl QueryPartition for IndexCache {
|
||||
// before). Chunked, not collected — the map is never copied whole.
|
||||
const CHUNK: usize = 4096;
|
||||
let mut entries = kmers.iter();
|
||||
let mut pending: Vec<(CanonicalKmer, &Vec<KmerDesc>)> = Vec::with_capacity(CHUNK);
|
||||
let mut pending: Vec<(CanonicalKmer, &[KmerDesc])> = Vec::with_capacity(CHUNK);
|
||||
let mut batch: Vec<CanonicalKmer> = Vec::with_capacity(CHUNK);
|
||||
let mut found: Vec<Option<usize>> = Vec::with_capacity(CHUNK);
|
||||
loop {
|
||||
pending.clear();
|
||||
pending.extend(entries.by_ref().take(CHUNK).map(|(k, d)| (*k, d)));
|
||||
pending.extend(entries.by_ref().take(CHUNK));
|
||||
if pending.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -87,8 +87,8 @@ fn duplicated_sequence_broadcasts_to_both_seq_indices() {
|
||||
let batch = batch_from_fasta(&fasta, K, 1);
|
||||
|
||||
let shared = batch.by_partition[0]
|
||||
.values()
|
||||
.find(|descs| descs.iter().any(|d| d.seq_idx == 0) && descs.iter().any(|d| d.seq_idx == 1));
|
||||
.iter()
|
||||
.find(|(_, descs)| descs.iter().any(|d| d.seq_idx == 0) && descs.iter().any(|d| d.seq_idx == 1));
|
||||
|
||||
assert!(
|
||||
shared.is_some(),
|
||||
@@ -119,9 +119,23 @@ fn partition_routing_is_a_pure_function_of_the_kmer() {
|
||||
|
||||
// No k-mer key appears in more than one partition's map.
|
||||
let mut seen: std::collections::HashSet<CanonicalKmer> = std::collections::HashSet::new();
|
||||
for map in &batch.by_partition {
|
||||
for kmer in map.keys() {
|
||||
assert!(seen.insert(*kmer), "k-mer routed to more than one partition");
|
||||
for part in &batch.by_partition {
|
||||
for (kmer, _) in part.iter() {
|
||||
assert!(seen.insert(kmer), "k-mer routed to more than one partition");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn occurrences_of_a_kmer_come_out_in_query_order() {
|
||||
// Grouping sorts occurrences; within a k-mer they must stay ordered by
|
||||
// (seq_idx, pos) — the order the former insertion-ordered `Vec` had, which
|
||||
// downstream output ordering relied on.
|
||||
let fasta = format!(">r1\n{SEQ}\n>r2\n{SEQ}\n");
|
||||
let batch = batch_from_fasta(&fasta, K, 1);
|
||||
for (_, descs) in batch.by_partition[0].iter() {
|
||||
assert!(descs.windows(2).all(|w| (w[0].seq_idx, w[0].pos) < (w[1].seq_idx, w[1].pos)));
|
||||
}
|
||||
let occ: usize = batch.by_partition[0].iter().map(|(_, d)| d.len()).sum();
|
||||
assert_eq!(occ as u64, total_occurrences(&batch));
|
||||
}
|
||||
@@ -64,12 +64,11 @@ fn query_partition_with_missing_index_dir_returns_default_stats() {
|
||||
let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index");
|
||||
let cache = IndexCache::new(Arc::new(index), Some(vec![0]));
|
||||
|
||||
let mut kmers: HashMap<CanonicalKmer, Vec<KmerDesc>> = HashMap::new();
|
||||
// Any well-formed canonical k-mer works here — the call must return
|
||||
// before ever attempting an MPHF lookup, since the partition has zero
|
||||
// layers.
|
||||
let kmer = CanonicalKmer::from_raw_unchecked(0u64);
|
||||
kmers.insert(kmer, vec![KmerDesc { seq_idx: 0, pos: 0 }]);
|
||||
let kmers = PartitionKmers::from_occurrences(vec![(kmer, KmerDesc { seq_idx: 0, pos: 0 })]);
|
||||
|
||||
let stats = cache.query_partition_with(0, &kmers, 1, |_event| {
|
||||
panic!("on_event must not be called: no index was built");
|
||||
@@ -94,7 +93,7 @@ fn query_partition_with_empty_kmers_is_a_noop() {
|
||||
let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index");
|
||||
let cache = IndexCache::new(Arc::new(index), Some(vec![0]));
|
||||
|
||||
let kmers: HashMap<CanonicalKmer, Vec<KmerDesc>> = HashMap::new();
|
||||
let kmers = PartitionKmers::empty();
|
||||
let stats = cache.query_partition_with(0, &kmers, 1, |_event| {
|
||||
panic!("on_event must not be called on an empty kmer map");
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user