Implement batch lookup and chunked data management

Introduces support for batched MPHF lookups, chunked data handling via `SlottedKmers`, and a comprehensive benchmarking suite to measure performance across the indexing and querying pipeline.

This includes new layer functionality for batch operations, chunking logic in the query layer, and scripts to measure wall time and resource usage for various commands. Documentation related to specific implementation details has also been cleaned up.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:14:51 +02:00
1 parent 8b87078478
commit fe38424682
10 files changed
+520 -30

No files matched your search

+39 -7
View File
@@ -134,16 +134,48 @@ impl QueryPartition for IndexCache {
let mut by_layer: Vec<HashMap<usize, &Vec<KmerDesc>>> =
(0..n_layer).map(|_| HashMap::new()).collect();
for (kmer, descs) in kmers {
stats.n_unique_kmers += 1;
// Layer-major over fixed-size chunks of the (unordered) kmer map:
// each layer sees the chunk's still-unresolved kmers as one batch, so
// its MPHF lookups are prefetched together; a kmer found in an
// earlier layer never reaches the later ones (first hit wins, as
// before). Chunked, not collected — the map is never copied whole.
const CHUNK: usize = 4096;
let mut entries = kmers.iter();
let mut pending: Vec<(CanonicalKmer, &Vec<KmerDesc>)> = Vec::with_capacity(CHUNK);
let mut batch: Vec<CanonicalKmer> = Vec::with_capacity(CHUNK);
let mut found: Vec<Option<usize>> = Vec::with_capacity(CHUNK);
loop {
pending.clear();
pending.extend(entries.by_ref().take(CHUNK).map(|(k, d)| (*k, d)));
if pending.is_empty() {
break;
}
stats.n_unique_kmers += pending.len();
for l in 0..n_layer {
stats.n_mphf_calls += 1;
if let Some(slot) = self.find_in_layer(part_idx, l, *kmer) {
by_layer[l].insert(slot, descs);
on_event(QueryHit::Found(descs));
stats.n_hits += 1;
if pending.is_empty() {
break;
}
let layer = self
.get_layer(part_idx, l)
.expect("layer within n_layer(part_idx)");
batch.clear();
batch.extend(pending.iter().map(|&(k, _)| k));
stats.n_mphf_calls += batch.len();
layer.find_slot_batch(&batch, &mut found);
let mut i = 0;
pending.retain(|&(_, descs)| {
let hit = found[i];
i += 1;
match hit {
Some(slot) => {
by_layer[l].insert(slot, descs);
on_event(QueryHit::Found(descs));
stats.n_hits += 1;
false
}
None => true,
}
});
}
}