Introduce memory mapping for index files and sparse matrices

This change implements memory-mapped views for index readers and sparse matrices, enabling shared page-cache access. It also includes performance analysis confirming that the default chunk size cap negatively impacts parallelism for small inputs.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:17:09 +02:00
1 parent 09c80468b2
commit 093d81c359
4 files changed
+116 -42

No files matched your search

+18 -5
View File
@@ -25,6 +25,7 @@ use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use memmap2::Mmap;
use ndarray::{Array1, Array2};
use crate::traits::{BitPartials, ColumnWeights};
@@ -151,7 +152,9 @@ pub struct PersistentSparseBitMatrix {
singleton: PersistentFixedIntVec,
multi: PersistentFixedIntVec,
dict_offsets: EliasFano,
dict_values: Vec<u8>,
/// `dict_values.bin`, mapped (never copied to the heap); `None` when the
/// file is empty — no multi-genome set — since an empty file cannot be mapped.
dict_values: Option<Mmap>,
/// Row count (k-mer slots).
n: usize,
/// Column count (genomes).
@@ -167,7 +170,12 @@ impl PersistentSparseBitMatrix {
let singleton = PersistentFixedIntVec::open(&singleton_path(dir))?;
let multi = PersistentFixedIntVec::open(&multi_path(dir))?;
let dict_offsets = EliasFano::open(&dict_offsets_base(dir))?;
let dict_values = fs::read(dict_values_path(dir))?;
let dict_file = fs::File::open(dict_values_path(dir))?;
let dict_values = if dict_file.metadata()?.len() == 0 {
None
} else {
Some(unsafe { Mmap::map(&dict_file)? })
};
Ok(Self {
is_multi, singleton, multi, dict_offsets, dict_values,
n: meta.n, n_cols: meta.n_cols, n_distinct_multi: meta.n_distinct_multi,
@@ -179,6 +187,11 @@ impl PersistentSparseBitMatrix {
#[inline]
pub fn n_cols(&self) -> usize { self.n_cols }
#[inline]
fn dict_bytes(&self) -> &[u8] {
self.dict_values.as_deref().unwrap_or(&[])
}
/// Byte range of `dict_id`'s varint-encoded genome list within
/// `dict_values`.
#[inline]
@@ -187,7 +200,7 @@ impl PersistentSparseBitMatrix {
let end = if dict_id + 1 < self.n_distinct_multi {
self.dict_offsets.get(dict_id + 1) as usize
} else {
self.dict_values.len()
self.dict_bytes().len()
};
(start, end)
}
@@ -232,7 +245,7 @@ impl PersistentSparseBitMatrix {
let (start, end) = self.dict_entry_range(dict_id);
let mut p = start;
while p < end {
f(read_varint(&self.dict_values, &mut p) as usize);
f(read_varint(self.dict_bytes(), &mut p) as usize);
}
} else {
let pos = self.is_multi.rank0(slot) as usize;
@@ -287,7 +300,7 @@ impl PersistentSparseBitMatrix {
present.clear();
let mut p = start;
while p < end {
present.push(read_varint(&self.dict_values, &mut p) as usize);
present.push(read_varint(self.dict_bytes(), &mut p) as usize);
}
for &g in &present {
counts[g] += w;