feat: implement in-memory column builders and refactor merge partitioning
Introduce `MemBitVecBuilder` and `MemCompactIntVecBuilder` to construct columns in RAM before writing, reducing the number of open file descriptors during merge operations. Refactor `partition_merge.rs` to use these new builders via `add_mem_col`, decoupling live memory mappings from source counts to prevent `SIGBUS` errors. - Add `mem_builder` module with `MemBitVecBuilder` and `MemCompactIntVecBuilder` structs for in-memory column construction and PCIV file writing. - Extend `MatrixBuilder` with `add_mem_col` and `n_cols` methods, updating `ColBuilder` enum to support `MemBit` and `MemInt` variants. - Refactor `partition_merge.rs` to use `add_mem_col` instead of `add_col`, introducing `run_pass2` for batch processing and optimizing destination layer handling. - Update `format.rs` with `pciv_index_layout`, `write_pciv_header`, and `write_pciv_tail` functions to manage PCIV index layout and header/tail sections. - Modify `mmap_file.rs` to use `write_std_header` and introduce `write_built` for standardized file writing and flushing logic. - Make `MAGIC` constant public in `bitvec.rs` and update documentation for memory mapping alignment and structure. - Add test modules `tests/matrix_builder.rs` and `tests/mem_builder.rs` to verify column counts and byte-identical output of new builders against persistent versions. - Update `mkdocs.yml` navigation to include "Merge mmap audit (2026-11)" documentation entry. - Add implementation audit documentation in `DevDocMD/implementation/merge_mmap_audit.md` detailing memory mapping failures and refactoring plans. - Add merge testing scripts `scripts/smoke_test_merge.sh` and `scripts/bench_merge_oracle.sh` for validation and benchmarking. - Bump `obikmer` package version from 1.4.2 to 1.4.3 in `Cargo.toml`.
This commit is contained in:
1 parent
b35127e4e3
commit
1f199c69db
25 files changed
+1631
-687
No files matched your search
@@ -3,6 +3,7 @@ use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::bitvec::PersistentBitVecBuilder;
|
||||
use crate::mem_builder::MemBitVecBuilder;
|
||||
use crate::meta::MatrixMeta;
|
||||
use crate::tempbitvec::TempBitVec;
|
||||
use crate::tempintvec::TempCompactIntVec;
|
||||
@@ -43,6 +44,14 @@ impl PersistentBitMatrixBuilder {
|
||||
PersistentBitVecBuilder::new(self.n, &path)
|
||||
}
|
||||
|
||||
/// Same as [`add_col`](Self::add_col) — next column, `n_cols` + 1 — but
|
||||
/// the column is built in RAM and written once at `close`.
|
||||
pub fn add_mem_col(&mut self) -> MemBitVecBuilder {
|
||||
let path = col_path(&self.dir, self.n_cols);
|
||||
self.n_cols += 1;
|
||||
MemBitVecBuilder::new(self.n, &path)
|
||||
}
|
||||
|
||||
pub fn add_col_ones(&mut self) -> io::Result<PersistentBitVecBuilder> {
|
||||
let path = col_path(&self.dir, self.n_cols);
|
||||
self.n_cols += 1;
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::mmap_file::{building_path, create_mmap_file, open_mmap_file, publish_
|
||||
use crate::reader::PersistentCompactIntVec;
|
||||
use crate::views::{BitSliceIter, BitSliceView, IntSliceView};
|
||||
|
||||
const MAGIC: [u8; 4] = *b"PBIV";
|
||||
pub(crate) const MAGIC: [u8; 4] = *b"PBIV";
|
||||
|
||||
// Header: magic(4) + _pad(4) + n(8) = 16 bytes.
|
||||
// Data starts at offset 16, u64-aligned (mmap base is page-aligned, 16 % 8 == 0).
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::{self, BufWriter, Seek, SeekFrom, Write as _};
|
||||
use std::io::{self, BufWriter, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
pub const MAGIC: [u8; 4] = *b"PCIV";
|
||||
@@ -54,53 +54,63 @@ pub fn parse_index_entry(data: &[u8], base: usize, i: usize) -> (usize, usize) {
|
||||
// Sparse index target: ≤ 32 KB in L1 cache (16 B per entry → 2048 entries).
|
||||
pub const L1_INDEX_ENTRIES: usize = 2048;
|
||||
|
||||
/// Sparse-index `step` and entry count for `n_overflow` overflow entries:
|
||||
/// no index (`step == 0`) while the overflow section fits the L1 target.
|
||||
fn pciv_index_layout(n_overflow: usize) -> (usize, usize) {
|
||||
if n_overflow <= L1_INDEX_ENTRIES {
|
||||
(0, 0)
|
||||
} else {
|
||||
let step = n_overflow.div_ceil(L1_INDEX_ENTRIES);
|
||||
(step, n_overflow.div_ceil(step))
|
||||
}
|
||||
}
|
||||
|
||||
/// Write the 40-byte PCIV header for a vector of `n` slots with
|
||||
/// `n_overflow` overflow entries (index layout derived from it).
|
||||
pub(crate) fn write_pciv_header<W: Write>(w: &mut W, n: usize, n_overflow: usize) -> io::Result<()> {
|
||||
let (step, n_index) = pciv_index_layout(n_overflow);
|
||||
w.write_all(&MAGIC)?;
|
||||
w.write_all(&[0u8; 4])?;
|
||||
w.write_all(&(n as u64).to_le_bytes())?;
|
||||
w.write_all(&(n_overflow as u64).to_le_bytes())?;
|
||||
w.write_all(&(n_index as u64).to_le_bytes())?;
|
||||
w.write_all(&(step as u64).to_le_bytes())
|
||||
}
|
||||
|
||||
/// Write the sections that follow the primary bytes:
|
||||
/// overflow n_overflow × 12 B (slot: u64, value: u32), `entries` sorted by slot
|
||||
/// index n_index × 16 B (slot: u64, pos: u64) sparse index
|
||||
pub(crate) fn write_pciv_tail<W: Write>(w: &mut W, entries: &[(usize, u32)]) -> io::Result<()> {
|
||||
for &(slot, value) in entries {
|
||||
w.write_all(&(slot as u64).to_le_bytes())?;
|
||||
w.write_all(&value.to_le_bytes())?;
|
||||
}
|
||||
let (step, _) = pciv_index_layout(entries.len());
|
||||
if step > 0 {
|
||||
for (block, chunk) in entries.chunks(step).enumerate() {
|
||||
w.write_all(&(chunk[0].0 as u64).to_le_bytes())?;
|
||||
w.write_all(&((block * step) as u64).to_le_bytes())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Finalise a PCIV file whose placeholder header and primary section are already on disk.
|
||||
///
|
||||
/// Truncates the file to `HEADER_SIZE + n`, then appends:
|
||||
/// overflow n_overflow × 12 B (slot: u64, value: u32) sorted by slot
|
||||
/// index n_index × 16 B (slot: u64, pos: u64) sparse index
|
||||
/// and overwrites the placeholder header at offset 0.
|
||||
/// Truncates the file to `HEADER_SIZE + n`, appends the overflow and index
|
||||
/// sections ([`write_pciv_tail`]) and overwrites the placeholder header at
|
||||
/// offset 0 ([`write_pciv_header`]).
|
||||
pub fn finalize_pciv(path: &Path, n: usize, entries: &[(usize, u32)]) -> io::Result<()> {
|
||||
let n_overflow = entries.len();
|
||||
|
||||
let file = OpenOptions::new().read(true).write(true).open(path)?;
|
||||
file.set_len((HEADER_SIZE + n) as u64)?;
|
||||
|
||||
let mut w = BufWriter::new(file);
|
||||
w.seek(SeekFrom::End(0))?;
|
||||
|
||||
for &(slot, value) in entries {
|
||||
w.write_all(&(slot as u64).to_le_bytes())?;
|
||||
w.write_all(&value.to_le_bytes())?;
|
||||
}
|
||||
|
||||
let step: usize = if n_overflow <= L1_INDEX_ENTRIES {
|
||||
0
|
||||
} else {
|
||||
n_overflow.div_ceil(L1_INDEX_ENTRIES)
|
||||
};
|
||||
|
||||
let n_index: usize = if step > 0 {
|
||||
let count = n_overflow.div_ceil(step);
|
||||
for (block, chunk) in entries.chunks(step).enumerate() {
|
||||
let slot = chunk[0].0 as u64;
|
||||
let pos = (block * step) as u64;
|
||||
w.write_all(&slot.to_le_bytes())?;
|
||||
w.write_all(&pos.to_le_bytes())?;
|
||||
}
|
||||
count
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
write_pciv_tail(&mut w, entries)?;
|
||||
w.flush()?;
|
||||
|
||||
let mut file = w.into_inner().map_err(|e| e.into_error())?;
|
||||
file.seek(SeekFrom::Start(0))?;
|
||||
file.write_all(&MAGIC)?;
|
||||
file.write_all(&[0u8; 4])?;
|
||||
file.write_all(&(n as u64).to_le_bytes())?;
|
||||
file.write_all(&(n_overflow as u64).to_le_bytes())?;
|
||||
file.write_all(&(n_index as u64).to_le_bytes())?;
|
||||
file.write_all(&(step as u64).to_le_bytes())?;
|
||||
write_pciv_header(&mut file, n, entries.len())?;
|
||||
file.flush()
|
||||
}
|
||||
@@ -11,6 +11,7 @@ use crate::bitmatrix::{
|
||||
use crate::builder::PersistentCompactIntVecBuilder;
|
||||
use crate::colgroup::{ColGroup, MatrixGroupOps, chunked_presence_count};
|
||||
use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE};
|
||||
use crate::mem_builder::MemCompactIntVecBuilder;
|
||||
use crate::meta::MatrixMeta;
|
||||
use crate::reader::PersistentCompactIntVec;
|
||||
use crate::sparse_intmatrix::{PersistentSparseCompactIntMatrix, is_present as sparse_is_present};
|
||||
@@ -717,6 +718,14 @@ impl PersistentCompactIntMatrixBuilder {
|
||||
PersistentCompactIntVecBuilder::new(self.n, &path)
|
||||
}
|
||||
|
||||
/// Same as [`add_col`](Self::add_col) — next column, `n_cols` + 1 — but
|
||||
/// the column is built in RAM and written once at `close`.
|
||||
pub fn add_mem_col(&mut self) -> MemCompactIntVecBuilder {
|
||||
let path = col_path(&self.dir, self.n_cols);
|
||||
self.n_cols += 1;
|
||||
MemCompactIntVecBuilder::new(self.n, &path)
|
||||
}
|
||||
|
||||
pub fn add_col_from(&mut self, src: &TempCompactIntVec) -> io::Result<()> {
|
||||
src.make_persistent(&col_path(&self.dir, self.n_cols))?;
|
||||
self.n_cols += 1;
|
||||
|
||||
@@ -9,6 +9,7 @@ mod int_group_ops;
|
||||
mod intmatrix;
|
||||
mod layer_meta;
|
||||
mod matrix_builder;
|
||||
mod mem_builder;
|
||||
mod meta;
|
||||
mod mmap_file;
|
||||
mod rankselect;
|
||||
@@ -35,6 +36,7 @@ pub use intmatrix::{
|
||||
};
|
||||
|
||||
pub use layer_meta::LayerMeta;
|
||||
pub use mem_builder::{MemBitVecBuilder, MemCompactIntVecBuilder};
|
||||
pub use mmap_file::{building_path, publish_built};
|
||||
pub use matrix_builder::{ColBuilder, MatrixBuilder};
|
||||
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
|
||||
|
||||
@@ -10,13 +10,19 @@ use std::path::Path;
|
||||
|
||||
use crate::{PersistentBitMatrixBuilder as BitMB, PersistentBitVecBuilder};
|
||||
use crate::{PersistentCompactIntMatrixBuilder as IntMB, PersistentCompactIntVecBuilder};
|
||||
use crate::{TempBitVec, TempCompactIntVec};
|
||||
use crate::{MemBitVecBuilder, MemCompactIntVecBuilder, TempBitVec, TempCompactIntVec};
|
||||
|
||||
// ── ColBuilder ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// One column being built. `Bit`/`Int` write through a mmap of a sparse
|
||||
/// file as values are set; `MemBit`/`MemInt` keep the column in RAM and
|
||||
/// write it once at `close` (same file, byte for byte — see
|
||||
/// [`MemBitVecBuilder`]).
|
||||
pub enum ColBuilder {
|
||||
Bit(PersistentBitVecBuilder),
|
||||
Int(PersistentCompactIntVecBuilder),
|
||||
MemBit(MemBitVecBuilder),
|
||||
MemInt(MemCompactIntVecBuilder),
|
||||
}
|
||||
|
||||
impl ColBuilder {
|
||||
@@ -24,6 +30,8 @@ impl ColBuilder {
|
||||
match self {
|
||||
ColBuilder::Bit(b) => b.set(slot, value > 0),
|
||||
ColBuilder::Int(b) => b.set(slot, value),
|
||||
ColBuilder::MemBit(b) => b.set(slot, value > 0),
|
||||
ColBuilder::MemInt(b) => b.set(slot, value),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +39,8 @@ impl ColBuilder {
|
||||
match self {
|
||||
ColBuilder::Bit(b) => b.close(),
|
||||
ColBuilder::Int(b) => b.close(),
|
||||
ColBuilder::MemBit(b) => b.close(),
|
||||
ColBuilder::MemInt(b) => b.close(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -64,6 +74,15 @@ impl MatrixBuilder {
|
||||
})
|
||||
}
|
||||
|
||||
/// Number of columns created so far (absent + data columns, including
|
||||
/// those inherited from a resumed directory).
|
||||
pub fn n_cols(&self) -> usize {
|
||||
match self {
|
||||
MatrixBuilder::Bit(b) => b.n_cols(),
|
||||
MatrixBuilder::Int(b) => b.n_cols(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a column with no data written (all-zero/false) — for columns
|
||||
/// absent from a given source (e.g. destination-only genomes not
|
||||
/// contributed by the source being merged in).
|
||||
@@ -81,6 +100,17 @@ impl MatrixBuilder {
|
||||
})
|
||||
}
|
||||
|
||||
/// Same as [`add_col`](Self::add_col) — next column, `n_cols` + 1 — but
|
||||
/// the column is built in RAM and written once, sequentially, at
|
||||
/// `close`: no sparse file, no writable mapping. Costs `n / 8` bytes
|
||||
/// (presence) or `n` bytes + overflow (counts) of RAM until closed.
|
||||
pub fn add_mem_col(&mut self) -> ColBuilder {
|
||||
match self {
|
||||
MatrixBuilder::Bit(b) => ColBuilder::MemBit(b.add_mem_col()),
|
||||
MatrixBuilder::Int(b) => ColBuilder::MemInt(b.add_mem_col()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a column already computed as a bit vector.
|
||||
pub fn add_col_from_bit(&mut self, src: &TempBitVec) -> io::Result<()> {
|
||||
match self {
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
//! In-memory column builders: the content of a column is built in RAM and
|
||||
//! written once, sequentially, when the builder is closed.
|
||||
//!
|
||||
//! They produce exactly the same file, byte for byte, as
|
||||
//! [`PersistentBitVecBuilder`](crate::PersistentBitVecBuilder) /
|
||||
//! [`PersistentCompactIntVecBuilder`](crate::PersistentCompactIntVecBuilder),
|
||||
//! but never create a sparse file nor a writable mapping: a block the file
|
||||
//! system cannot allocate (full disk, quota, network FS) is reported as an
|
||||
//! `io::Error` by `close`, instead of a SIGBUS on the page fault that first
|
||||
//! touches it. The price is that the whole column lives in RAM until
|
||||
//! `close`: `n / 8` bytes for bits, `n` bytes (+ overflow map) for counts.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::bitvec::{n_words, MAGIC as PBIV_MAGIC};
|
||||
use crate::format::{write_pciv_header, write_pciv_tail};
|
||||
use crate::mmap_file::{write_built, write_std_header};
|
||||
|
||||
// ── Bits (PBIV) ───────────────────────────────────────────────────────────────
|
||||
|
||||
/// In-memory counterpart of `PersistentBitVecBuilder` (`set` + `close` only).
|
||||
pub struct MemBitVecBuilder {
|
||||
words: Vec<u64>,
|
||||
n: usize,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl MemBitVecBuilder {
|
||||
/// All-false vector of `n` bits, to be written at `path` by `close`.
|
||||
pub fn new(n: usize, path: &Path) -> Self {
|
||||
Self { words: vec![0u64; n_words(n)], n, path: path.to_path_buf() }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.n
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.n == 0
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn set(&mut self, slot: usize, value: bool) {
|
||||
debug_assert!(slot < self.n);
|
||||
let bit = 1u64 << (slot & 63);
|
||||
if value {
|
||||
self.words[slot >> 6] |= bit;
|
||||
} else {
|
||||
self.words[slot >> 6] &= !bit;
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes the PBIV file — header, then the words in native byte order,
|
||||
/// exactly as the mmap-backed builder lays them out — and publishes it.
|
||||
pub fn close(self) -> io::Result<()> {
|
||||
let Self { words, n, path } = self;
|
||||
write_built(&path, |w| {
|
||||
write_std_header(w, PBIV_MAGIC, [0u8; 4], n)?;
|
||||
for word in &words {
|
||||
w.write_all(&word.to_ne_bytes())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── Compact ints (PCIV) ───────────────────────────────────────────────────────
|
||||
|
||||
/// In-memory counterpart of `PersistentCompactIntVecBuilder` (`set` +
|
||||
/// `close` only). Same encoding: one byte per slot, values ≥ 255 stored as
|
||||
/// the sentinel 255 plus an overflow entry.
|
||||
pub struct MemCompactIntVecBuilder {
|
||||
primary: Vec<u8>,
|
||||
overflow: HashMap<usize, u32>,
|
||||
n: usize,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl MemCompactIntVecBuilder {
|
||||
/// All-zero vector of `n` slots, to be written at `path` by `close`.
|
||||
pub fn new(n: usize, path: &Path) -> Self {
|
||||
Self { primary: vec![0u8; n], overflow: HashMap::new(), n, path: path.to_path_buf() }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.n
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.n == 0
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn set(&mut self, slot: usize, value: u32) {
|
||||
if value < 255 {
|
||||
self.primary[slot] = value as u8;
|
||||
self.overflow.remove(&slot);
|
||||
} else {
|
||||
self.primary[slot] = 255;
|
||||
self.overflow.insert(slot, value);
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes the PCIV file — header, primary bytes, overflow entries sorted
|
||||
/// by slot, sparse index — and publishes it.
|
||||
pub fn close(self) -> io::Result<()> {
|
||||
let Self { primary, overflow, n, path } = self;
|
||||
let mut entries: Vec<(usize, u32)> = overflow.into_iter().collect();
|
||||
entries.sort_unstable_by_key(|&(slot, _)| slot);
|
||||
write_built(&path, |w| {
|
||||
write_pciv_header(w, n, entries.len())?;
|
||||
w.write_all(&primary)?;
|
||||
write_pciv_tail(w, &entries)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@
|
||||
//! between formats and aren't forced into a shared shape.
|
||||
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Seek, SeekFrom, Write as _};
|
||||
use std::io::{self, BufWriter, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use memmap2::{Mmap, MmapMut};
|
||||
@@ -37,6 +37,37 @@ pub fn publish_built(path: &Path) -> io::Result<()> {
|
||||
fs::rename(building_path(path), path)
|
||||
}
|
||||
|
||||
/// Writes the crate-wide standard header: `magic` + `pad` + `n as u64` (LE).
|
||||
pub(crate) fn write_std_header<W: Write>(w: &mut W, magic: [u8; 4], pad: [u8; 4], n: usize) -> io::Result<()> {
|
||||
w.write_all(&magic)?;
|
||||
w.write_all(&pad)?;
|
||||
w.write_all(&(n as u64).to_le_bytes())
|
||||
}
|
||||
|
||||
/// Writes a whole file sequentially through `write` into
|
||||
/// [`building_path`]`(path)`, flushes it, then publishes it at `path`. No
|
||||
/// sparse file and no writable mapping: an allocation failure on the file
|
||||
/// system (full disk, quota) surfaces as an `io::Error` from a `write`,
|
||||
/// never as a SIGBUS on a page fault.
|
||||
///
|
||||
/// Deliberately no `sync_all`: one forced sync per file made the merge
|
||||
/// ~8× slower (thousands of column files; measured 2026-11-17 on macOS),
|
||||
/// and the mmap builders it replaces never synced either. Durability is the
|
||||
/// OS's, like the rest of an index; the `rename` still guarantees that no
|
||||
/// partially written file is ever published. Caveat: on a write-back
|
||||
/// network FS an allocation error may only be reported at `close`, which
|
||||
/// `File`'s drop ignores.
|
||||
pub(crate) fn write_built(
|
||||
path: &Path,
|
||||
write: impl FnOnce(&mut BufWriter<File>) -> io::Result<()>,
|
||||
) -> io::Result<()> {
|
||||
let mut w = BufWriter::new(File::create(building_path(path))?);
|
||||
write(&mut w)?;
|
||||
w.flush()?;
|
||||
drop(w.into_inner().map_err(|e| e.into_error())?);
|
||||
publish_built(path)
|
||||
}
|
||||
|
||||
/// Creates (or truncates) [`building_path`]`(path)` — the caller publishes it
|
||||
/// with [`publish_built`] once complete — writes the standard header — `magic` +
|
||||
/// `pad` (4 bytes right after `magic`; usually all-zero, but
|
||||
@@ -60,9 +91,7 @@ pub(crate) fn create_mmap_file(
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(building_path(path))?;
|
||||
file.write_all(&magic)?;
|
||||
file.write_all(&pad)?;
|
||||
file.write_all(&(n as u64).to_le_bytes())?;
|
||||
write_std_header(&mut file, magic, pad, n)?;
|
||||
if extra_header_zeros > 0 {
|
||||
file.write_all(&vec![0u8; extra_header_zeros])?;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
use tempfile::tempdir;
|
||||
|
||||
use crate::MatrixBuilder;
|
||||
|
||||
#[test]
|
||||
fn n_cols_counts_absent_data_and_resumed_columns() {
|
||||
for presence in [true, false] {
|
||||
let dir = tempdir().unwrap();
|
||||
let data = dir.path().join("m");
|
||||
|
||||
let mut mb = MatrixBuilder::new(presence, 5, &data).unwrap();
|
||||
assert_eq!(mb.n_cols(), 0);
|
||||
mb.add_absent_col().unwrap();
|
||||
assert_eq!(mb.n_cols(), 1);
|
||||
let mut c = mb.add_col().unwrap();
|
||||
c.set_val(2, 7);
|
||||
c.close().unwrap();
|
||||
assert_eq!(mb.n_cols(), 2);
|
||||
mb.close().unwrap();
|
||||
|
||||
let mut mb = MatrixBuilder::resume(presence, &data).unwrap();
|
||||
assert_eq!(mb.n_cols(), 2, "presence={presence}");
|
||||
mb.add_absent_col().unwrap();
|
||||
assert_eq!(mb.n_cols(), 3);
|
||||
mb.close().unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
use std::fs;
|
||||
|
||||
use tempfile::tempdir;
|
||||
|
||||
use crate::{
|
||||
MatrixBuilder, MemBitVecBuilder, MemCompactIntVecBuilder, PersistentBitMatrix,
|
||||
PersistentBitVecBuilder, PersistentCompactIntVecBuilder, PersistentIntMatrix,
|
||||
};
|
||||
|
||||
const SIZES: [usize; 6] = [0, 1, 63, 64, 65, 10_000];
|
||||
|
||||
/// Deterministic xorshift64* — no RNG dependency needed for test data.
|
||||
struct Rng(u64);
|
||||
impl Rng {
|
||||
fn next(&mut self) -> u64 {
|
||||
self.0 ^= self.0 >> 12;
|
||||
self.0 ^= self.0 << 25;
|
||||
self.0 ^= self.0 >> 27;
|
||||
self.0.wrapping_mul(0x2545_F491_4F6C_DD1D)
|
||||
}
|
||||
}
|
||||
|
||||
/// `(slot, value)` writes in random order, with overwrites; for counts,
|
||||
/// about a third of the values are ≥ 255 (overflow), enough at n = 10 000
|
||||
/// to exceed `L1_INDEX_ENTRIES` and exercise the sparse index section.
|
||||
fn writes(n: usize, seed: u64, max_small: u64) -> Vec<(usize, u32)> {
|
||||
let mut rng = Rng(seed);
|
||||
(0..2 * n)
|
||||
.map(|_| {
|
||||
let slot = (rng.next() % n as u64) as usize;
|
||||
let r = rng.next();
|
||||
let value = match r % 3 {
|
||||
0 => 255 + (r >> 8) as u32 % 100_000, // overflow
|
||||
1 => 0, // also clears a previous overflow
|
||||
_ => (r >> 8) as u32 % max_small as u32,
|
||||
};
|
||||
(slot, value)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mem_bitvec_is_byte_identical_to_persistent() {
|
||||
for n in SIZES {
|
||||
let dir = tempdir().unwrap();
|
||||
let (pp, mp) = (dir.path().join("p.pbiv"), dir.path().join("m.pbiv"));
|
||||
let mut p = PersistentBitVecBuilder::new(n, &pp).unwrap();
|
||||
let mut m = MemBitVecBuilder::new(n, &mp);
|
||||
for (slot, v) in writes(n, 0x9E37_79B9 + n as u64, 2) {
|
||||
p.set(slot, v > 0);
|
||||
m.set(slot, v > 0);
|
||||
}
|
||||
p.close().unwrap();
|
||||
m.close().unwrap();
|
||||
assert_eq!(fs::read(&pp).unwrap(), fs::read(&mp).unwrap(), "n = {n}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mem_compact_int_is_byte_identical_to_persistent() {
|
||||
for n in SIZES {
|
||||
let dir = tempdir().unwrap();
|
||||
let (pp, mp) = (dir.path().join("p.pciv"), dir.path().join("m.pciv"));
|
||||
let mut p = PersistentCompactIntVecBuilder::new(n, &pp).unwrap();
|
||||
let mut m = MemCompactIntVecBuilder::new(n, &mp);
|
||||
for (slot, v) in writes(n, 0xC2B2_AE35 + n as u64, 255) {
|
||||
p.set(slot, v);
|
||||
m.set(slot, v);
|
||||
}
|
||||
p.close().unwrap();
|
||||
m.close().unwrap();
|
||||
assert_eq!(fs::read(&pp).unwrap(), fs::read(&mp).unwrap(), "n = {n}");
|
||||
}
|
||||
}
|
||||
|
||||
/// `add_mem_col` columns, interleaved with absent columns and continued
|
||||
/// across a `resume`, read back through the regular matrix readers.
|
||||
#[test]
|
||||
fn add_mem_col_matrix_reads_back() {
|
||||
for presence in [true, false] {
|
||||
let n = 100;
|
||||
let dir = tempdir().unwrap();
|
||||
let data = dir.path().join(if presence { "presence" } else { "counts" });
|
||||
let value = |c: usize, slot: usize| -> u32 {
|
||||
if (slot + c).is_multiple_of(3) { (slot * 7 + c) as u32 % 400 + 1 } else { 0 }
|
||||
};
|
||||
|
||||
let mut mb = MatrixBuilder::new(presence, n, &data).unwrap();
|
||||
mb.add_absent_col().unwrap();
|
||||
let mut c1 = mb.add_mem_col();
|
||||
(0..n).for_each(|s| c1.set_val(s, value(1, s)));
|
||||
c1.close().unwrap();
|
||||
mb.close().unwrap();
|
||||
|
||||
let mut mb = MatrixBuilder::resume(presence, &data).unwrap();
|
||||
assert_eq!(mb.n_cols(), 2);
|
||||
let mut c2 = mb.add_mem_col();
|
||||
(0..n).for_each(|s| c2.set_val(s, value(2, s)));
|
||||
c2.close().unwrap();
|
||||
assert_eq!(mb.n_cols(), 3);
|
||||
mb.close().unwrap();
|
||||
|
||||
for s in 0..n {
|
||||
let expect = [0, value(1, s), value(2, s)];
|
||||
if presence {
|
||||
let m = PersistentBitMatrix::open(dir.path()).unwrap();
|
||||
assert_eq!(m.n_cols(), 3);
|
||||
let want: Vec<bool> = expect.iter().map(|&v| v > 0).collect();
|
||||
assert_eq!(&*m.row(s), want.as_slice(), "slot {s}");
|
||||
} else {
|
||||
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||
assert_eq!(m.n_cols(), 3);
|
||||
assert_eq!(&*m.row(s), &expect, "slot {s}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ mod eliasfano;
|
||||
mod fixedintvec;
|
||||
mod int_group_ops;
|
||||
mod intmatrix;
|
||||
mod matrix_builder;
|
||||
mod mem_builder;
|
||||
mod rankselect;
|
||||
mod sparse;
|
||||
mod sparse_intmatrix;
|
||||
|
||||
Reference in new issue
Block a user