Introduce memory mapping for index files and sparse matrices
This change implements memory-mapped views for index readers and sparse matrices, enabling shared page-cache access. It also includes performance analysis confirming that the default chunk size cap negatively impacts parallelism for small inputs.
This commit is contained in:
1 parent
09c80468b2
commit
093d81c359
4 files changed
+116
-42
No files matched your search
@@ -25,6 +25,7 @@ use std::fs;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use memmap2::Mmap;
|
||||
use ndarray::{Array1, Array2};
|
||||
|
||||
use crate::traits::{BitPartials, ColumnWeights};
|
||||
@@ -151,7 +152,9 @@ pub struct PersistentSparseBitMatrix {
|
||||
singleton: PersistentFixedIntVec,
|
||||
multi: PersistentFixedIntVec,
|
||||
dict_offsets: EliasFano,
|
||||
dict_values: Vec<u8>,
|
||||
/// `dict_values.bin`, mapped (never copied to the heap); `None` when the
|
||||
/// file is empty — no multi-genome set — since an empty file cannot be mapped.
|
||||
dict_values: Option<Mmap>,
|
||||
/// Row count (k-mer slots).
|
||||
n: usize,
|
||||
/// Column count (genomes).
|
||||
@@ -167,7 +170,12 @@ impl PersistentSparseBitMatrix {
|
||||
let singleton = PersistentFixedIntVec::open(&singleton_path(dir))?;
|
||||
let multi = PersistentFixedIntVec::open(&multi_path(dir))?;
|
||||
let dict_offsets = EliasFano::open(&dict_offsets_base(dir))?;
|
||||
let dict_values = fs::read(dict_values_path(dir))?;
|
||||
let dict_file = fs::File::open(dict_values_path(dir))?;
|
||||
let dict_values = if dict_file.metadata()?.len() == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(unsafe { Mmap::map(&dict_file)? })
|
||||
};
|
||||
Ok(Self {
|
||||
is_multi, singleton, multi, dict_offsets, dict_values,
|
||||
n: meta.n, n_cols: meta.n_cols, n_distinct_multi: meta.n_distinct_multi,
|
||||
@@ -179,6 +187,11 @@ impl PersistentSparseBitMatrix {
|
||||
#[inline]
|
||||
pub fn n_cols(&self) -> usize { self.n_cols }
|
||||
|
||||
#[inline]
|
||||
fn dict_bytes(&self) -> &[u8] {
|
||||
self.dict_values.as_deref().unwrap_or(&[])
|
||||
}
|
||||
|
||||
/// Byte range of `dict_id`'s varint-encoded genome list within
|
||||
/// `dict_values`.
|
||||
#[inline]
|
||||
@@ -187,7 +200,7 @@ impl PersistentSparseBitMatrix {
|
||||
let end = if dict_id + 1 < self.n_distinct_multi {
|
||||
self.dict_offsets.get(dict_id + 1) as usize
|
||||
} else {
|
||||
self.dict_values.len()
|
||||
self.dict_bytes().len()
|
||||
};
|
||||
(start, end)
|
||||
}
|
||||
@@ -232,7 +245,7 @@ impl PersistentSparseBitMatrix {
|
||||
let (start, end) = self.dict_entry_range(dict_id);
|
||||
let mut p = start;
|
||||
while p < end {
|
||||
f(read_varint(&self.dict_values, &mut p) as usize);
|
||||
f(read_varint(self.dict_bytes(), &mut p) as usize);
|
||||
}
|
||||
} else {
|
||||
let pos = self.is_multi.rank0(slot) as usize;
|
||||
@@ -287,7 +300,7 @@ impl PersistentSparseBitMatrix {
|
||||
present.clear();
|
||||
let mut p = start;
|
||||
while p < end {
|
||||
present.push(read_varint(&self.dict_values, &mut p) as usize);
|
||||
present.push(read_varint(self.dict_bytes(), &mut p) as usize);
|
||||
}
|
||||
for &g in &present {
|
||||
counts[g] += w;
|
||||
|
||||
Reference in new issue
Block a user