feat: implement in-memory column builders and refactor merge partitioning
Introduce `MemBitVecBuilder` and `MemCompactIntVecBuilder` to construct columns in RAM before writing, reducing the number of open file descriptors during merge operations. Refactor `partition_merge.rs` to use these new builders via `add_mem_col`, decoupling live memory mappings from source counts to prevent `SIGBUS` errors. - Add `mem_builder` module with `MemBitVecBuilder` and `MemCompactIntVecBuilder` structs for in-memory column construction and PCIV file writing. - Extend `MatrixBuilder` with `add_mem_col` and `n_cols` methods, updating `ColBuilder` enum to support `MemBit` and `MemInt` variants. - Refactor `partition_merge.rs` to use `add_mem_col` instead of `add_col`, introducing `run_pass2` for batch processing and optimizing destination layer handling. - Update `format.rs` with `pciv_index_layout`, `write_pciv_header`, and `write_pciv_tail` functions to manage PCIV index layout and header/tail sections. - Modify `mmap_file.rs` to use `write_std_header` and introduce `write_built` for standardized file writing and flushing logic. - Make `MAGIC` constant public in `bitvec.rs` and update documentation for memory mapping alignment and structure. - Add test modules `tests/matrix_builder.rs` and `tests/mem_builder.rs` to verify column counts and byte-identical output of new builders against persistent versions. - Update `mkdocs.yml` navigation to include "Merge mmap audit (2026-11)" documentation entry. - Add implementation audit documentation in `DevDocMD/implementation/merge_mmap_audit.md` detailing memory mapping failures and refactoring plans. - Add merge testing scripts `scripts/smoke_test_merge.sh` and `scripts/bench_merge_oracle.sh` for validation and benchmarking. - Bump `obikmer` package version from 1.4.2 to 1.4.3 in `Cargo.toml`.
This commit is contained in:
1 parent
b35127e4e3
commit
1f199c69db
25 files changed
+1631
-687
No files matched your search
Generated
+1
-2
@@ -1537,7 +1537,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "obikmer"
|
||||
version = "1.4.2"
|
||||
version = "1.4.3"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"csv",
|
||||
@@ -1582,7 +1582,6 @@ dependencies = [
|
||||
"obiskio",
|
||||
"obisys",
|
||||
"rayon",
|
||||
"tempfile",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::bitvec::PersistentBitVecBuilder;
|
||||
use crate::mem_builder::MemBitVecBuilder;
|
||||
use crate::meta::MatrixMeta;
|
||||
use crate::tempbitvec::TempBitVec;
|
||||
use crate::tempintvec::TempCompactIntVec;
|
||||
@@ -43,6 +44,14 @@ impl PersistentBitMatrixBuilder {
|
||||
PersistentBitVecBuilder::new(self.n, &path)
|
||||
}
|
||||
|
||||
/// Same as [`add_col`](Self::add_col) — next column, `n_cols` + 1 — but
|
||||
/// the column is built in RAM and written once at `close`.
|
||||
pub fn add_mem_col(&mut self) -> MemBitVecBuilder {
|
||||
let path = col_path(&self.dir, self.n_cols);
|
||||
self.n_cols += 1;
|
||||
MemBitVecBuilder::new(self.n, &path)
|
||||
}
|
||||
|
||||
pub fn add_col_ones(&mut self) -> io::Result<PersistentBitVecBuilder> {
|
||||
let path = col_path(&self.dir, self.n_cols);
|
||||
self.n_cols += 1;
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::mmap_file::{building_path, create_mmap_file, open_mmap_file, publish_
|
||||
use crate::reader::PersistentCompactIntVec;
|
||||
use crate::views::{BitSliceIter, BitSliceView, IntSliceView};
|
||||
|
||||
const MAGIC: [u8; 4] = *b"PBIV";
|
||||
pub(crate) const MAGIC: [u8; 4] = *b"PBIV";
|
||||
|
||||
// Header: magic(4) + _pad(4) + n(8) = 16 bytes.
|
||||
// Data starts at offset 16, u64-aligned (mmap base is page-aligned, 16 % 8 == 0).
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::{self, BufWriter, Seek, SeekFrom, Write as _};
|
||||
use std::io::{self, BufWriter, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
pub const MAGIC: [u8; 4] = *b"PCIV";
|
||||
@@ -54,53 +54,63 @@ pub fn parse_index_entry(data: &[u8], base: usize, i: usize) -> (usize, usize) {
|
||||
// Sparse index target: ≤ 32 KB in L1 cache (16 B per entry → 2048 entries).
|
||||
pub const L1_INDEX_ENTRIES: usize = 2048;
|
||||
|
||||
/// Sparse-index `step` and entry count for `n_overflow` overflow entries:
|
||||
/// no index (`step == 0`) while the overflow section fits the L1 target.
|
||||
fn pciv_index_layout(n_overflow: usize) -> (usize, usize) {
|
||||
if n_overflow <= L1_INDEX_ENTRIES {
|
||||
(0, 0)
|
||||
} else {
|
||||
let step = n_overflow.div_ceil(L1_INDEX_ENTRIES);
|
||||
(step, n_overflow.div_ceil(step))
|
||||
}
|
||||
}
|
||||
|
||||
/// Write the 40-byte PCIV header for a vector of `n` slots with
|
||||
/// `n_overflow` overflow entries (index layout derived from it).
|
||||
pub(crate) fn write_pciv_header<W: Write>(w: &mut W, n: usize, n_overflow: usize) -> io::Result<()> {
|
||||
let (step, n_index) = pciv_index_layout(n_overflow);
|
||||
w.write_all(&MAGIC)?;
|
||||
w.write_all(&[0u8; 4])?;
|
||||
w.write_all(&(n as u64).to_le_bytes())?;
|
||||
w.write_all(&(n_overflow as u64).to_le_bytes())?;
|
||||
w.write_all(&(n_index as u64).to_le_bytes())?;
|
||||
w.write_all(&(step as u64).to_le_bytes())
|
||||
}
|
||||
|
||||
/// Write the sections that follow the primary bytes:
|
||||
/// overflow n_overflow × 12 B (slot: u64, value: u32), `entries` sorted by slot
|
||||
/// index n_index × 16 B (slot: u64, pos: u64) sparse index
|
||||
pub(crate) fn write_pciv_tail<W: Write>(w: &mut W, entries: &[(usize, u32)]) -> io::Result<()> {
|
||||
for &(slot, value) in entries {
|
||||
w.write_all(&(slot as u64).to_le_bytes())?;
|
||||
w.write_all(&value.to_le_bytes())?;
|
||||
}
|
||||
let (step, _) = pciv_index_layout(entries.len());
|
||||
if step > 0 {
|
||||
for (block, chunk) in entries.chunks(step).enumerate() {
|
||||
w.write_all(&(chunk[0].0 as u64).to_le_bytes())?;
|
||||
w.write_all(&((block * step) as u64).to_le_bytes())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Finalise a PCIV file whose placeholder header and primary section are already on disk.
|
||||
///
|
||||
/// Truncates the file to `HEADER_SIZE + n`, then appends:
|
||||
/// overflow n_overflow × 12 B (slot: u64, value: u32) sorted by slot
|
||||
/// index n_index × 16 B (slot: u64, pos: u64) sparse index
|
||||
/// and overwrites the placeholder header at offset 0.
|
||||
/// Truncates the file to `HEADER_SIZE + n`, appends the overflow and index
|
||||
/// sections ([`write_pciv_tail`]) and overwrites the placeholder header at
|
||||
/// offset 0 ([`write_pciv_header`]).
|
||||
pub fn finalize_pciv(path: &Path, n: usize, entries: &[(usize, u32)]) -> io::Result<()> {
|
||||
let n_overflow = entries.len();
|
||||
|
||||
let file = OpenOptions::new().read(true).write(true).open(path)?;
|
||||
file.set_len((HEADER_SIZE + n) as u64)?;
|
||||
|
||||
let mut w = BufWriter::new(file);
|
||||
w.seek(SeekFrom::End(0))?;
|
||||
|
||||
for &(slot, value) in entries {
|
||||
w.write_all(&(slot as u64).to_le_bytes())?;
|
||||
w.write_all(&value.to_le_bytes())?;
|
||||
}
|
||||
|
||||
let step: usize = if n_overflow <= L1_INDEX_ENTRIES {
|
||||
0
|
||||
} else {
|
||||
n_overflow.div_ceil(L1_INDEX_ENTRIES)
|
||||
};
|
||||
|
||||
let n_index: usize = if step > 0 {
|
||||
let count = n_overflow.div_ceil(step);
|
||||
for (block, chunk) in entries.chunks(step).enumerate() {
|
||||
let slot = chunk[0].0 as u64;
|
||||
let pos = (block * step) as u64;
|
||||
w.write_all(&slot.to_le_bytes())?;
|
||||
w.write_all(&pos.to_le_bytes())?;
|
||||
}
|
||||
count
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
write_pciv_tail(&mut w, entries)?;
|
||||
w.flush()?;
|
||||
|
||||
let mut file = w.into_inner().map_err(|e| e.into_error())?;
|
||||
file.seek(SeekFrom::Start(0))?;
|
||||
file.write_all(&MAGIC)?;
|
||||
file.write_all(&[0u8; 4])?;
|
||||
file.write_all(&(n as u64).to_le_bytes())?;
|
||||
file.write_all(&(n_overflow as u64).to_le_bytes())?;
|
||||
file.write_all(&(n_index as u64).to_le_bytes())?;
|
||||
file.write_all(&(step as u64).to_le_bytes())?;
|
||||
write_pciv_header(&mut file, n, entries.len())?;
|
||||
file.flush()
|
||||
}
|
||||
@@ -11,6 +11,7 @@ use crate::bitmatrix::{
|
||||
use crate::builder::PersistentCompactIntVecBuilder;
|
||||
use crate::colgroup::{ColGroup, MatrixGroupOps, chunked_presence_count};
|
||||
use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE};
|
||||
use crate::mem_builder::MemCompactIntVecBuilder;
|
||||
use crate::meta::MatrixMeta;
|
||||
use crate::reader::PersistentCompactIntVec;
|
||||
use crate::sparse_intmatrix::{PersistentSparseCompactIntMatrix, is_present as sparse_is_present};
|
||||
@@ -717,6 +718,14 @@ impl PersistentCompactIntMatrixBuilder {
|
||||
PersistentCompactIntVecBuilder::new(self.n, &path)
|
||||
}
|
||||
|
||||
/// Same as [`add_col`](Self::add_col) — next column, `n_cols` + 1 — but
|
||||
/// the column is built in RAM and written once at `close`.
|
||||
pub fn add_mem_col(&mut self) -> MemCompactIntVecBuilder {
|
||||
let path = col_path(&self.dir, self.n_cols);
|
||||
self.n_cols += 1;
|
||||
MemCompactIntVecBuilder::new(self.n, &path)
|
||||
}
|
||||
|
||||
pub fn add_col_from(&mut self, src: &TempCompactIntVec) -> io::Result<()> {
|
||||
src.make_persistent(&col_path(&self.dir, self.n_cols))?;
|
||||
self.n_cols += 1;
|
||||
|
||||
@@ -9,6 +9,7 @@ mod int_group_ops;
|
||||
mod intmatrix;
|
||||
mod layer_meta;
|
||||
mod matrix_builder;
|
||||
mod mem_builder;
|
||||
mod meta;
|
||||
mod mmap_file;
|
||||
mod rankselect;
|
||||
@@ -35,6 +36,7 @@ pub use intmatrix::{
|
||||
};
|
||||
|
||||
pub use layer_meta::LayerMeta;
|
||||
pub use mem_builder::{MemBitVecBuilder, MemCompactIntVecBuilder};
|
||||
pub use mmap_file::{building_path, publish_built};
|
||||
pub use matrix_builder::{ColBuilder, MatrixBuilder};
|
||||
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
|
||||
|
||||
@@ -10,13 +10,19 @@ use std::path::Path;
|
||||
|
||||
use crate::{PersistentBitMatrixBuilder as BitMB, PersistentBitVecBuilder};
|
||||
use crate::{PersistentCompactIntMatrixBuilder as IntMB, PersistentCompactIntVecBuilder};
|
||||
use crate::{TempBitVec, TempCompactIntVec};
|
||||
use crate::{MemBitVecBuilder, MemCompactIntVecBuilder, TempBitVec, TempCompactIntVec};
|
||||
|
||||
// ── ColBuilder ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// One column being built. `Bit`/`Int` write through a mmap of a sparse
|
||||
/// file as values are set; `MemBit`/`MemInt` keep the column in RAM and
|
||||
/// write it once at `close` (same file, byte for byte — see
|
||||
/// [`MemBitVecBuilder`]).
|
||||
pub enum ColBuilder {
|
||||
Bit(PersistentBitVecBuilder),
|
||||
Int(PersistentCompactIntVecBuilder),
|
||||
MemBit(MemBitVecBuilder),
|
||||
MemInt(MemCompactIntVecBuilder),
|
||||
}
|
||||
|
||||
impl ColBuilder {
|
||||
@@ -24,6 +30,8 @@ impl ColBuilder {
|
||||
match self {
|
||||
ColBuilder::Bit(b) => b.set(slot, value > 0),
|
||||
ColBuilder::Int(b) => b.set(slot, value),
|
||||
ColBuilder::MemBit(b) => b.set(slot, value > 0),
|
||||
ColBuilder::MemInt(b) => b.set(slot, value),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +39,8 @@ impl ColBuilder {
|
||||
match self {
|
||||
ColBuilder::Bit(b) => b.close(),
|
||||
ColBuilder::Int(b) => b.close(),
|
||||
ColBuilder::MemBit(b) => b.close(),
|
||||
ColBuilder::MemInt(b) => b.close(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -64,6 +74,15 @@ impl MatrixBuilder {
|
||||
})
|
||||
}
|
||||
|
||||
/// Number of columns created so far (absent + data columns, including
|
||||
/// those inherited from a resumed directory).
|
||||
pub fn n_cols(&self) -> usize {
|
||||
match self {
|
||||
MatrixBuilder::Bit(b) => b.n_cols(),
|
||||
MatrixBuilder::Int(b) => b.n_cols(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a column with no data written (all-zero/false) — for columns
|
||||
/// absent from a given source (e.g. destination-only genomes not
|
||||
/// contributed by the source being merged in).
|
||||
@@ -81,6 +100,17 @@ impl MatrixBuilder {
|
||||
})
|
||||
}
|
||||
|
||||
/// Same as [`add_col`](Self::add_col) — next column, `n_cols` + 1 — but
|
||||
/// the column is built in RAM and written once, sequentially, at
|
||||
/// `close`: no sparse file, no writable mapping. Costs `n / 8` bytes
|
||||
/// (presence) or `n` bytes + overflow (counts) of RAM until closed.
|
||||
pub fn add_mem_col(&mut self) -> ColBuilder {
|
||||
match self {
|
||||
MatrixBuilder::Bit(b) => ColBuilder::MemBit(b.add_mem_col()),
|
||||
MatrixBuilder::Int(b) => ColBuilder::MemInt(b.add_mem_col()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a column already computed as a bit vector.
|
||||
pub fn add_col_from_bit(&mut self, src: &TempBitVec) -> io::Result<()> {
|
||||
match self {
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
//! In-memory column builders: the content of a column is built in RAM and
|
||||
//! written once, sequentially, when the builder is closed.
|
||||
//!
|
||||
//! They produce exactly the same file, byte for byte, as
|
||||
//! [`PersistentBitVecBuilder`](crate::PersistentBitVecBuilder) /
|
||||
//! [`PersistentCompactIntVecBuilder`](crate::PersistentCompactIntVecBuilder),
|
||||
//! but never create a sparse file nor a writable mapping: a block the file
|
||||
//! system cannot allocate (full disk, quota, network FS) is reported as an
|
||||
//! `io::Error` by `close`, instead of a SIGBUS on the page fault that first
|
||||
//! touches it. The price is that the whole column lives in RAM until
|
||||
//! `close`: `n / 8` bytes for bits, `n` bytes (+ overflow map) for counts.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::bitvec::{n_words, MAGIC as PBIV_MAGIC};
|
||||
use crate::format::{write_pciv_header, write_pciv_tail};
|
||||
use crate::mmap_file::{write_built, write_std_header};
|
||||
|
||||
// ── Bits (PBIV) ───────────────────────────────────────────────────────────────
|
||||
|
||||
/// In-memory counterpart of `PersistentBitVecBuilder` (`set` + `close` only).
|
||||
pub struct MemBitVecBuilder {
|
||||
words: Vec<u64>,
|
||||
n: usize,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl MemBitVecBuilder {
|
||||
/// All-false vector of `n` bits, to be written at `path` by `close`.
|
||||
pub fn new(n: usize, path: &Path) -> Self {
|
||||
Self { words: vec![0u64; n_words(n)], n, path: path.to_path_buf() }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.n
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.n == 0
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn set(&mut self, slot: usize, value: bool) {
|
||||
debug_assert!(slot < self.n);
|
||||
let bit = 1u64 << (slot & 63);
|
||||
if value {
|
||||
self.words[slot >> 6] |= bit;
|
||||
} else {
|
||||
self.words[slot >> 6] &= !bit;
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes the PBIV file — header, then the words in native byte order,
|
||||
/// exactly as the mmap-backed builder lays them out — and publishes it.
|
||||
pub fn close(self) -> io::Result<()> {
|
||||
let Self { words, n, path } = self;
|
||||
write_built(&path, |w| {
|
||||
write_std_header(w, PBIV_MAGIC, [0u8; 4], n)?;
|
||||
for word in &words {
|
||||
w.write_all(&word.to_ne_bytes())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── Compact ints (PCIV) ───────────────────────────────────────────────────────
|
||||
|
||||
/// In-memory counterpart of `PersistentCompactIntVecBuilder` (`set` +
|
||||
/// `close` only). Same encoding: one byte per slot, values ≥ 255 stored as
|
||||
/// the sentinel 255 plus an overflow entry.
|
||||
pub struct MemCompactIntVecBuilder {
|
||||
primary: Vec<u8>,
|
||||
overflow: HashMap<usize, u32>,
|
||||
n: usize,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl MemCompactIntVecBuilder {
|
||||
/// All-zero vector of `n` slots, to be written at `path` by `close`.
|
||||
pub fn new(n: usize, path: &Path) -> Self {
|
||||
Self { primary: vec![0u8; n], overflow: HashMap::new(), n, path: path.to_path_buf() }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.n
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.n == 0
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn set(&mut self, slot: usize, value: u32) {
|
||||
if value < 255 {
|
||||
self.primary[slot] = value as u8;
|
||||
self.overflow.remove(&slot);
|
||||
} else {
|
||||
self.primary[slot] = 255;
|
||||
self.overflow.insert(slot, value);
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes the PCIV file — header, primary bytes, overflow entries sorted
|
||||
/// by slot, sparse index — and publishes it.
|
||||
pub fn close(self) -> io::Result<()> {
|
||||
let Self { primary, overflow, n, path } = self;
|
||||
let mut entries: Vec<(usize, u32)> = overflow.into_iter().collect();
|
||||
entries.sort_unstable_by_key(|&(slot, _)| slot);
|
||||
write_built(&path, |w| {
|
||||
write_pciv_header(w, n, entries.len())?;
|
||||
w.write_all(&primary)?;
|
||||
write_pciv_tail(w, &entries)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@
|
||||
//! between formats and aren't forced into a shared shape.
|
||||
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Seek, SeekFrom, Write as _};
|
||||
use std::io::{self, BufWriter, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use memmap2::{Mmap, MmapMut};
|
||||
@@ -37,6 +37,37 @@ pub fn publish_built(path: &Path) -> io::Result<()> {
|
||||
fs::rename(building_path(path), path)
|
||||
}
|
||||
|
||||
/// Writes the crate-wide standard header: `magic` + `pad` + `n as u64` (LE).
|
||||
pub(crate) fn write_std_header<W: Write>(w: &mut W, magic: [u8; 4], pad: [u8; 4], n: usize) -> io::Result<()> {
|
||||
w.write_all(&magic)?;
|
||||
w.write_all(&pad)?;
|
||||
w.write_all(&(n as u64).to_le_bytes())
|
||||
}
|
||||
|
||||
/// Writes a whole file sequentially through `write` into
|
||||
/// [`building_path`]`(path)`, flushes it, then publishes it at `path`. No
|
||||
/// sparse file and no writable mapping: an allocation failure on the file
|
||||
/// system (full disk, quota) surfaces as an `io::Error` from a `write`,
|
||||
/// never as a SIGBUS on a page fault.
|
||||
///
|
||||
/// Deliberately no `sync_all`: one forced sync per file made the merge
|
||||
/// ~8× slower (thousands of column files; measured 2026-11-17 on macOS),
|
||||
/// and the mmap builders it replaces never synced either. Durability is the
|
||||
/// OS's, like the rest of an index; the `rename` still guarantees that no
|
||||
/// partially written file is ever published. Caveat: on a write-back
|
||||
/// network FS an allocation error may only be reported at `close`, which
|
||||
/// `File`'s drop ignores.
|
||||
pub(crate) fn write_built(
|
||||
path: &Path,
|
||||
write: impl FnOnce(&mut BufWriter<File>) -> io::Result<()>,
|
||||
) -> io::Result<()> {
|
||||
let mut w = BufWriter::new(File::create(building_path(path))?);
|
||||
write(&mut w)?;
|
||||
w.flush()?;
|
||||
drop(w.into_inner().map_err(|e| e.into_error())?);
|
||||
publish_built(path)
|
||||
}
|
||||
|
||||
/// Creates (or truncates) [`building_path`]`(path)` — the caller publishes it
|
||||
/// with [`publish_built`] once complete — writes the standard header — `magic` +
|
||||
/// `pad` (4 bytes right after `magic`; usually all-zero, but
|
||||
@@ -60,9 +91,7 @@ pub(crate) fn create_mmap_file(
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(building_path(path))?;
|
||||
file.write_all(&magic)?;
|
||||
file.write_all(&pad)?;
|
||||
file.write_all(&(n as u64).to_le_bytes())?;
|
||||
write_std_header(&mut file, magic, pad, n)?;
|
||||
if extra_header_zeros > 0 {
|
||||
file.write_all(&vec![0u8; extra_header_zeros])?;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
use tempfile::tempdir;
|
||||
|
||||
use crate::MatrixBuilder;
|
||||
|
||||
#[test]
|
||||
fn n_cols_counts_absent_data_and_resumed_columns() {
|
||||
for presence in [true, false] {
|
||||
let dir = tempdir().unwrap();
|
||||
let data = dir.path().join("m");
|
||||
|
||||
let mut mb = MatrixBuilder::new(presence, 5, &data).unwrap();
|
||||
assert_eq!(mb.n_cols(), 0);
|
||||
mb.add_absent_col().unwrap();
|
||||
assert_eq!(mb.n_cols(), 1);
|
||||
let mut c = mb.add_col().unwrap();
|
||||
c.set_val(2, 7);
|
||||
c.close().unwrap();
|
||||
assert_eq!(mb.n_cols(), 2);
|
||||
mb.close().unwrap();
|
||||
|
||||
let mut mb = MatrixBuilder::resume(presence, &data).unwrap();
|
||||
assert_eq!(mb.n_cols(), 2, "presence={presence}");
|
||||
mb.add_absent_col().unwrap();
|
||||
assert_eq!(mb.n_cols(), 3);
|
||||
mb.close().unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
use std::fs;
|
||||
|
||||
use tempfile::tempdir;
|
||||
|
||||
use crate::{
|
||||
MatrixBuilder, MemBitVecBuilder, MemCompactIntVecBuilder, PersistentBitMatrix,
|
||||
PersistentBitVecBuilder, PersistentCompactIntVecBuilder, PersistentIntMatrix,
|
||||
};
|
||||
|
||||
const SIZES: [usize; 6] = [0, 1, 63, 64, 65, 10_000];
|
||||
|
||||
/// Deterministic xorshift64* — no RNG dependency needed for test data.
|
||||
struct Rng(u64);
|
||||
impl Rng {
|
||||
fn next(&mut self) -> u64 {
|
||||
self.0 ^= self.0 >> 12;
|
||||
self.0 ^= self.0 << 25;
|
||||
self.0 ^= self.0 >> 27;
|
||||
self.0.wrapping_mul(0x2545_F491_4F6C_DD1D)
|
||||
}
|
||||
}
|
||||
|
||||
/// `(slot, value)` writes in random order, with overwrites; for counts,
|
||||
/// about a third of the values are ≥ 255 (overflow), enough at n = 10 000
|
||||
/// to exceed `L1_INDEX_ENTRIES` and exercise the sparse index section.
|
||||
fn writes(n: usize, seed: u64, max_small: u64) -> Vec<(usize, u32)> {
|
||||
let mut rng = Rng(seed);
|
||||
(0..2 * n)
|
||||
.map(|_| {
|
||||
let slot = (rng.next() % n as u64) as usize;
|
||||
let r = rng.next();
|
||||
let value = match r % 3 {
|
||||
0 => 255 + (r >> 8) as u32 % 100_000, // overflow
|
||||
1 => 0, // also clears a previous overflow
|
||||
_ => (r >> 8) as u32 % max_small as u32,
|
||||
};
|
||||
(slot, value)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mem_bitvec_is_byte_identical_to_persistent() {
|
||||
for n in SIZES {
|
||||
let dir = tempdir().unwrap();
|
||||
let (pp, mp) = (dir.path().join("p.pbiv"), dir.path().join("m.pbiv"));
|
||||
let mut p = PersistentBitVecBuilder::new(n, &pp).unwrap();
|
||||
let mut m = MemBitVecBuilder::new(n, &mp);
|
||||
for (slot, v) in writes(n, 0x9E37_79B9 + n as u64, 2) {
|
||||
p.set(slot, v > 0);
|
||||
m.set(slot, v > 0);
|
||||
}
|
||||
p.close().unwrap();
|
||||
m.close().unwrap();
|
||||
assert_eq!(fs::read(&pp).unwrap(), fs::read(&mp).unwrap(), "n = {n}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mem_compact_int_is_byte_identical_to_persistent() {
|
||||
for n in SIZES {
|
||||
let dir = tempdir().unwrap();
|
||||
let (pp, mp) = (dir.path().join("p.pciv"), dir.path().join("m.pciv"));
|
||||
let mut p = PersistentCompactIntVecBuilder::new(n, &pp).unwrap();
|
||||
let mut m = MemCompactIntVecBuilder::new(n, &mp);
|
||||
for (slot, v) in writes(n, 0xC2B2_AE35 + n as u64, 255) {
|
||||
p.set(slot, v);
|
||||
m.set(slot, v);
|
||||
}
|
||||
p.close().unwrap();
|
||||
m.close().unwrap();
|
||||
assert_eq!(fs::read(&pp).unwrap(), fs::read(&mp).unwrap(), "n = {n}");
|
||||
}
|
||||
}
|
||||
|
||||
/// `add_mem_col` columns, interleaved with absent columns and continued
|
||||
/// across a `resume`, read back through the regular matrix readers.
|
||||
#[test]
|
||||
fn add_mem_col_matrix_reads_back() {
|
||||
for presence in [true, false] {
|
||||
let n = 100;
|
||||
let dir = tempdir().unwrap();
|
||||
let data = dir.path().join(if presence { "presence" } else { "counts" });
|
||||
let value = |c: usize, slot: usize| -> u32 {
|
||||
if (slot + c).is_multiple_of(3) { (slot * 7 + c) as u32 % 400 + 1 } else { 0 }
|
||||
};
|
||||
|
||||
let mut mb = MatrixBuilder::new(presence, n, &data).unwrap();
|
||||
mb.add_absent_col().unwrap();
|
||||
let mut c1 = mb.add_mem_col();
|
||||
(0..n).for_each(|s| c1.set_val(s, value(1, s)));
|
||||
c1.close().unwrap();
|
||||
mb.close().unwrap();
|
||||
|
||||
let mut mb = MatrixBuilder::resume(presence, &data).unwrap();
|
||||
assert_eq!(mb.n_cols(), 2);
|
||||
let mut c2 = mb.add_mem_col();
|
||||
(0..n).for_each(|s| c2.set_val(s, value(2, s)));
|
||||
c2.close().unwrap();
|
||||
assert_eq!(mb.n_cols(), 3);
|
||||
mb.close().unwrap();
|
||||
|
||||
for s in 0..n {
|
||||
let expect = [0, value(1, s), value(2, s)];
|
||||
if presence {
|
||||
let m = PersistentBitMatrix::open(dir.path()).unwrap();
|
||||
assert_eq!(m.n_cols(), 3);
|
||||
let want: Vec<bool> = expect.iter().map(|&v| v > 0).collect();
|
||||
assert_eq!(&*m.row(s), want.as_slice(), "slot {s}");
|
||||
} else {
|
||||
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||
assert_eq!(m.n_cols(), 3);
|
||||
assert_eq!(&*m.row(s), &expect, "slot {s}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ mod eliasfano;
|
||||
mod fixedintvec;
|
||||
mod int_group_ops;
|
||||
mod intmatrix;
|
||||
mod matrix_builder;
|
||||
mod mem_builder;
|
||||
mod rankselect;
|
||||
mod sparse;
|
||||
mod sparse_intmatrix;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "obikmer"
|
||||
version = "1.4.2"
|
||||
version = "1.4.3"
|
||||
edition = "2024"
|
||||
|
||||
[[bin]]
|
||||
|
||||
@@ -15,5 +15,3 @@ obisys = { path = "../obisys", default-features = false }
|
||||
rayon = "1"
|
||||
tracing = "0.1.44"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
@@ -1,605 +0,0 @@
|
||||
//! Merging a source partition's new layer into a destination partition's
|
||||
//! de Bruijn graph union (pass 1) then column fill (pass 2). Free function,
|
||||
//! not a `KmerIndex` method — `KmerIndex` stays the stateless
|
||||
//! `Index { Partition { Layer } }` data model; this is one of the
|
||||
//! algorithms that reads/writes through it, living here in `obikmerge`
|
||||
//! instead.
|
||||
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use obipipeline::{
|
||||
Pipeline, PipelineError, PipelineSender, SharedFlatFn, Stage, ThrottleGuard, WorkerPool,
|
||||
make_sink, make_source, make_transform, throttle,
|
||||
};
|
||||
use tracing::debug;
|
||||
|
||||
use obicompactvec::{ColBuilder, MatrixBuilder};
|
||||
use obikindex::layer::{IndexMode, KmerLayer, TypedLayer};
|
||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||
use obikindexer::{build_graph, materialize_layer};
|
||||
use obikseq::CanonicalKmer;
|
||||
use obiskio::UnitigFileReader;
|
||||
|
||||
use crate::MergeMode;
|
||||
|
||||
// `MatrixBuilder::new`/`resume` take a `presence: bool` — `MergeMode` maps
|
||||
// to it directly (`Presence` → `true`, `Count` → `false`).
|
||||
fn presence(mode: MergeMode) -> bool {
|
||||
mode == MergeMode::Presence
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod matrix_builder_tests {
|
||||
use tempfile::tempdir;
|
||||
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
|
||||
|
||||
use super::{ColBuilder, MatrixBuilder};
|
||||
|
||||
/// Mirrors `merge_partition`'s "new layer" setup: absent (dst-genome)
|
||||
/// columns get no data, then source columns are filled — all through one
|
||||
/// continuous `MatrixBuilder` session, closed once at the end.
|
||||
#[test]
|
||||
fn new_layer_absent_then_source_columns_presence() {
|
||||
let dir = tempdir().unwrap();
|
||||
let data_dir = dir.path().join("presence");
|
||||
let mut mb = MatrixBuilder::new(true, 3, &data_dir).unwrap();
|
||||
|
||||
// Two absent (dst-genome) columns.
|
||||
mb.add_absent_col().unwrap();
|
||||
mb.add_absent_col().unwrap();
|
||||
|
||||
// One source column, filled like pass 2 would.
|
||||
let mut col = mb.add_col().unwrap();
|
||||
match &mut col {
|
||||
ColBuilder::Bit(b) => {
|
||||
b.set(0, true);
|
||||
b.set(1, false);
|
||||
b.set(2, true);
|
||||
}
|
||||
ColBuilder::Int(_) => unreachable!(),
|
||||
}
|
||||
col.close().unwrap();
|
||||
mb.close().unwrap();
|
||||
|
||||
let m = PersistentBitMatrix::open(dir.path()).unwrap();
|
||||
assert_eq!(m.n_cols(), 3);
|
||||
assert_eq!(&*m.row(0), &[false, false, true]);
|
||||
assert_eq!(&*m.row(1), &[false, false, false]);
|
||||
assert_eq!(&*m.row(2), &[false, false, true]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_layer_absent_then_source_columns_count() {
|
||||
let dir = tempdir().unwrap();
|
||||
let data_dir = dir.path().join("counts");
|
||||
let mut mb = MatrixBuilder::new(false, 2, &data_dir).unwrap();
|
||||
|
||||
mb.add_absent_col().unwrap();
|
||||
|
||||
let mut col = mb.add_col().unwrap();
|
||||
match &mut col {
|
||||
ColBuilder::Int(b) => {
|
||||
b.set(0, 7);
|
||||
b.set(1, 42);
|
||||
}
|
||||
ColBuilder::Bit(_) => unreachable!(),
|
||||
}
|
||||
col.close().unwrap();
|
||||
mb.close().unwrap();
|
||||
|
||||
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||
assert_eq!(m.n_cols(), 2);
|
||||
assert_eq!(&*m.row(0), &[0u32, 7]);
|
||||
assert_eq!(&*m.row(1), &[0u32, 42]);
|
||||
}
|
||||
|
||||
/// Mirrors `merge_partition`'s "existing dst layer" setup: an
|
||||
/// already-closed matrix (from a previous merge) gets more columns
|
||||
/// appended via `resume`, without callers ever seeing `col_path`/
|
||||
/// `MatrixMeta` — `n` itself is read back from the matrix directory.
|
||||
#[test]
|
||||
fn resume_appends_source_columns_to_existing_layer() {
|
||||
let dir = tempdir().unwrap();
|
||||
let data_dir = dir.path().join("presence");
|
||||
|
||||
// Previous merge: one dst-genome column already on disk.
|
||||
let mut mb0 = MatrixBuilder::new(true, 3, &data_dir).unwrap();
|
||||
mb0.add_absent_col().unwrap();
|
||||
mb0.close().unwrap();
|
||||
|
||||
// This merge: resume and append two more source columns.
|
||||
let mut mb = MatrixBuilder::resume(true, &data_dir).unwrap();
|
||||
for vals in [[true, false, true], [false, true, false]] {
|
||||
let mut col = mb.add_col().unwrap();
|
||||
match &mut col {
|
||||
ColBuilder::Bit(b) => {
|
||||
for (slot, v) in vals.into_iter().enumerate() {
|
||||
b.set(slot, v);
|
||||
}
|
||||
}
|
||||
ColBuilder::Int(_) => unreachable!(),
|
||||
}
|
||||
col.close().unwrap();
|
||||
}
|
||||
mb.close().unwrap();
|
||||
|
||||
let m = PersistentBitMatrix::open(dir.path()).unwrap();
|
||||
assert_eq!(m.n_cols(), 3);
|
||||
assert_eq!(&*m.row(0), &[false, true, false]);
|
||||
assert_eq!(&*m.row(1), &[false, false, true]);
|
||||
assert_eq!(&*m.row(2), &[false, true, false]);
|
||||
}
|
||||
}
|
||||
|
||||
// ── dst cross-layer lookup ──────────────────────────────────────────────────
|
||||
|
||||
/// Batch membership lookup across `dst_layers` — first layer that carries
|
||||
/// each kmer wins. Groups hits by layer index (as `(kmer batch index,
|
||||
/// slot)` pairs) rather than returning them in query order, so pass 2 can
|
||||
/// follow with one column-major write per `(layer, column)` instead of
|
||||
/// interleaving across layers kmer by kmer. The kmers with no hit in any
|
||||
/// layer come back as a separate index list, for the new-layer MPHF batch
|
||||
/// lookup on a miss.
|
||||
fn find_batch_grouped(
|
||||
layers: &[KmerLayer],
|
||||
kmers: &[CanonicalKmer],
|
||||
) -> (Vec<Vec<(usize, usize)>>, Vec<usize>) {
|
||||
let mut by_layer: Vec<Vec<(usize, usize)>> = vec![Vec::new(); layers.len()];
|
||||
let mut not_found: Vec<usize> = Vec::new();
|
||||
for (idx, &kmer) in kmers.iter().enumerate() {
|
||||
match layers
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find_map(|(li, l)| l.find_slot(kmer).map(|slot| (li, slot)))
|
||||
{
|
||||
Some((li, slot)) => by_layer[li].push((idx, slot)),
|
||||
None => not_found.push(idx),
|
||||
}
|
||||
}
|
||||
(by_layer, not_found)
|
||||
}
|
||||
|
||||
// ── merge_partition ────────────────────────────────────────────────────────
|
||||
|
||||
/// Merge `sources` into destination partition `i`.
|
||||
///
|
||||
/// Each entry in `sources` is `(index, n_genomes)` where `n_genomes` is the
|
||||
/// number of genome columns that source contributes. A merged index
|
||||
/// contributes more than one. The total new columns added to the
|
||||
/// destination is `sum(n_genomes)`.
|
||||
///
|
||||
/// `n_dst_genomes` is the number of genome columns already in the
|
||||
/// destination matrices (copied from the bootstrap source before this
|
||||
/// call).
|
||||
pub(crate) fn merge_partition(
|
||||
dst: &KmerIndex,
|
||||
i: usize,
|
||||
sources: &[(&KmerIndex, usize)],
|
||||
mode: MergeMode,
|
||||
n_dst_genomes: usize,
|
||||
block_bits: u8,
|
||||
evidence: &IndexMode,
|
||||
) -> OKIResult<usize> {
|
||||
let dst_partition = dst.partition(i)?;
|
||||
let n_dst_layers = dst_partition.n_layers();
|
||||
let n_src_total: usize = sources.iter().map(|(_, n)| *n).sum();
|
||||
|
||||
// First merge in presence mode: materialize implicit single-genome
|
||||
// presence matrices on existing layers (all slots true — every kmer in
|
||||
// those layers belongs to genome_0) so a resumed builder can append
|
||||
// source columns onto them below.
|
||||
if n_dst_genomes == 1 && mode == MergeMode::Presence {
|
||||
for l in 0..n_dst_layers {
|
||||
let layer = dst_partition.layer(l)?.open()?;
|
||||
TypedLayer::<()>::init_presence_matrix(layer.dir(), layer.n())?;
|
||||
}
|
||||
}
|
||||
|
||||
// dst's `0..n_dst_layers` layers and every source's own layers, for
|
||||
// this one partition — all opened the same direct way, not via
|
||||
// `IndexCache`. Every layer read here (dst's pre-existing ones, and
|
||||
// every source's) is just as stable for the duration of this call: dst
|
||||
// only ever gets a brand new layer *appended* below, never one of these
|
||||
// read from, so there's no "dst isn't stable" distinction between the
|
||||
// two loops — `IndexCache` still doesn't fit either one, for a single,
|
||||
// uniform reason: `obikindexer::build_graph`/`obipipeline::WorkerPool`
|
||||
// (pass 1/2 below) are plain-OS-thread pipelines requiring `Send + Sync
|
||||
// + 'static`, and `IndexCache<'_>` always borrows `&KmerIndex` — no
|
||||
// amount of logical stability changes that a borrowed cache can't cross
|
||||
// that boundary, only owned `KmerLayer`s can (each owns its own mmap'd
|
||||
// files, no reference back to the index it came from).
|
||||
let dst_layers: Vec<KmerLayer> = (0..n_dst_layers)
|
||||
.map(|l| dst_partition.layer(l)?.open())
|
||||
.collect::<OKIResult<_>>()?;
|
||||
let dst_layers = Arc::new(dst_layers);
|
||||
|
||||
let src_layers: Vec<Vec<Arc<KmerLayer>>> = sources
|
||||
.iter()
|
||||
.map(|(src, _)| -> OKIResult<Vec<Arc<KmerLayer>>> {
|
||||
let partition = src.partition(i)?;
|
||||
(0..partition.n_layers())
|
||||
.map(|l| partition.layer(l)?.open().map(Arc::new))
|
||||
.collect()
|
||||
})
|
||||
.collect::<OKIResult<_>>()?;
|
||||
|
||||
// ── Pass 1: pipeline — parallel file read + dst filter + graph fill ────
|
||||
//
|
||||
// Source : list of unitigs.bin paths (one per source × layer)
|
||||
// Flat : open file, emit Vec<CanonicalKmer> batches (BeeGFS parallel I/O)
|
||||
// Transform: filter via dst_layers — Arc<[KmerLayer]>: Sync
|
||||
// Sink : push new kmers into GraphDeBruijn (single thread, no locks needed)
|
||||
|
||||
let mut unitig_paths: Vec<PathBuf> = Vec::new();
|
||||
for layers in &src_layers {
|
||||
for layer in layers {
|
||||
let p = layer.unitigs_path();
|
||||
if p.exists() {
|
||||
unitig_paths.push(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let n_src_layers = unitig_paths.len();
|
||||
debug!("partition {i}: de Bruijn graph build start — {n_src_layers} source layer(s)");
|
||||
|
||||
const BATCH: usize = 4096;
|
||||
let n_workers = rayon::current_num_threads().min(16).max(4);
|
||||
// At most 2 files open simultaneously: keeps n_workers-2 workers free
|
||||
// for the Transform stage. Each open file monopolises one worker for the
|
||||
// full duration of its read, so this must stay well below n_workers.
|
||||
let max_open = 2;
|
||||
|
||||
let dst_filter = Arc::clone(&dst_layers);
|
||||
|
||||
let g = build_graph(
|
||||
unitig_paths.into_iter(),
|
||||
move |path: PathBuf, emit: &mut dyn FnMut(Vec<CanonicalKmer>)| -> OKIResult<()> {
|
||||
let reader = UnitigFileReader::open_sequential(&path)?;
|
||||
let mut batch: Vec<CanonicalKmer> = Vec::with_capacity(BATCH);
|
||||
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
|
||||
batch.push(kmer);
|
||||
if batch.len() == BATCH {
|
||||
emit(std::mem::replace(&mut batch, Vec::with_capacity(BATCH)));
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
emit(batch);
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
move |kmer| dst_filter.iter().all(|l| l.find_slot(kmer).is_none()),
|
||||
n_workers,
|
||||
max_open,
|
||||
)?;
|
||||
|
||||
let any_new = g.len() > 0;
|
||||
debug!(
|
||||
"partition {i}: de Bruijn graph done — {} new kmers",
|
||||
g.len()
|
||||
);
|
||||
|
||||
// Build new layer from de Bruijn graph if there are new kmers.
|
||||
let new_layer_idx = n_dst_layers;
|
||||
let new_layer_dir = KmerLayer::new(&dst_partition, new_layer_idx)
|
||||
.dir()
|
||||
.to_path_buf();
|
||||
|
||||
let n_new = if any_new {
|
||||
debug!("partition {i}: unitig traversal start — {} nodes", g.len());
|
||||
let n_nodes = materialize_layer(g, &new_layer_dir, block_bits, evidence)?;
|
||||
debug!("partition {i}: MPHF build done");
|
||||
n_nodes
|
||||
} else {
|
||||
drop(g);
|
||||
0
|
||||
};
|
||||
|
||||
let t_open = std::time::Instant::now();
|
||||
let new_mphf: Option<Arc<obikindex::layer::MphfOnly>> = if any_new {
|
||||
Some(Arc::new(obikindex::layer::MphfOnly::open(&new_layer_dir)?))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
debug!(
|
||||
"partition {i}: MPHF open in {:.3}s",
|
||||
t_open.elapsed().as_secs_f64()
|
||||
);
|
||||
|
||||
// ── Prepare matrix directories for the new layer ──────────────────────
|
||||
// Absent columns (dst genomes) get an all-zero/false column. Source-genome
|
||||
// columns are created as mutable builders for pass 2. `new_mb` is kept
|
||||
// open (not `close`d) until pass 2 has filled every source column, so its
|
||||
// `n_cols` bookkeeping stays in sync with the columns actually created.
|
||||
let (new_src_builders, new_mb): (Vec<ColBuilder>, Option<MatrixBuilder>) = if any_new {
|
||||
let data_dir = match mode {
|
||||
MergeMode::Presence => new_layer_dir.join("presence"),
|
||||
MergeMode::Count => new_layer_dir.join("counts"),
|
||||
};
|
||||
fs::create_dir_all(&data_dir)?;
|
||||
let mut mb = MatrixBuilder::new(presence(mode), n_new, &data_dir).map_err(OKIError::Io)?;
|
||||
for _ in 0..n_dst_genomes {
|
||||
mb.add_absent_col().map_err(OKIError::Io)?;
|
||||
}
|
||||
let cols = (0..n_src_total)
|
||||
.map(|_| mb.add_col().map_err(OKIError::Io))
|
||||
.collect::<OKIResult<Vec<_>>>()?;
|
||||
(cols, Some(mb))
|
||||
} else {
|
||||
(vec![], None)
|
||||
};
|
||||
|
||||
let t_builders = std::time::Instant::now();
|
||||
// Builders for existing layers: n_src_total per layer, resumed from
|
||||
// each layer's own matrix directory (already holding n_dst_genomes
|
||||
// columns from a previous merge). Columns land at
|
||||
// n_dst_genomes .. n_dst_genomes + n_src_total - 1.
|
||||
let mut exist_mbs: Vec<MatrixBuilder> = Vec::with_capacity(n_dst_layers);
|
||||
let mut exist_builders: Vec<Vec<ColBuilder>> = Vec::with_capacity(n_dst_layers);
|
||||
for l in 0..n_dst_layers {
|
||||
let layer_dir = dst_layers[l].dir().to_path_buf();
|
||||
let data_dir = match mode {
|
||||
MergeMode::Presence => layer_dir.join("presence"),
|
||||
MergeMode::Count => layer_dir.join("counts"),
|
||||
};
|
||||
let mut mb = MatrixBuilder::resume(presence(mode), &data_dir).map_err(OKIError::Io)?;
|
||||
let cols = (0..n_src_total)
|
||||
.map(|_| mb.add_col().map_err(OKIError::Io))
|
||||
.collect::<OKIResult<Vec<_>>>()?;
|
||||
exist_mbs.push(mb);
|
||||
exist_builders.push(cols);
|
||||
}
|
||||
|
||||
debug!(
|
||||
"partition {i}: builders ready in {:.3}s",
|
||||
t_builders.elapsed().as_secs_f64()
|
||||
);
|
||||
|
||||
// ── Pass 2: fill builders (pipeline) ─────────────────────────────────
|
||||
let t_pass2 = std::time::Instant::now();
|
||||
// Collect source items before the pipeline so errors propagate via ?
|
||||
// before any worker thread is spawned.
|
||||
let mut pass2_items: Vec<(usize, usize, PathBuf, Arc<KmerLayer>)> = Vec::new();
|
||||
{
|
||||
let mut col_offset = 0usize;
|
||||
for ((_, src_n), layers) in sources.iter().zip(src_layers.iter()) {
|
||||
for layer in layers {
|
||||
let unitigs_path = layer.unitigs_path();
|
||||
if unitigs_path.exists() {
|
||||
pass2_items.push((col_offset, *src_n, unitigs_path, Arc::clone(layer)));
|
||||
}
|
||||
}
|
||||
col_offset += src_n;
|
||||
}
|
||||
}
|
||||
|
||||
enum Pass2Data {
|
||||
SrcLayer((usize, usize, PathBuf, Arc<KmerLayer>, ThrottleGuard)),
|
||||
RawBatch((usize, usize, Arc<KmerLayer>, Vec<CanonicalKmer>)),
|
||||
/// One entry per `(dst layer, absolute column)` touched by this
|
||||
/// batch, each carrying every `(slot, value)` pair for that column
|
||||
/// together — column-major, so the sink below locks that column's
|
||||
/// `ColBuilder` once per batch and writes the whole run, instead of
|
||||
/// once per `(kmer, genome)` pair.
|
||||
WriteBatch(Vec<(Option<usize>, usize, Vec<(usize, u32)>)>),
|
||||
}
|
||||
|
||||
let exist_locked: Vec<Vec<Arc<Mutex<ColBuilder>>>> = exist_builders
|
||||
.into_iter()
|
||||
.map(|layer| layer.into_iter().map(|b| Arc::new(Mutex::new(b))).collect())
|
||||
.collect();
|
||||
let new_locked: Vec<Arc<Mutex<ColBuilder>>> = new_src_builders
|
||||
.into_iter()
|
||||
.map(|b| Arc::new(Mutex::new(b)))
|
||||
.collect();
|
||||
let exist_sink: Vec<Vec<Arc<Mutex<ColBuilder>>>> = exist_locked
|
||||
.iter()
|
||||
.map(|layer| layer.iter().map(Arc::clone).collect())
|
||||
.collect();
|
||||
let new_sink: Vec<Arc<Mutex<ColBuilder>>> = new_locked.iter().map(Arc::clone).collect();
|
||||
let dst_layers_t2 = Arc::clone(&dst_layers);
|
||||
let new_mphf_t2 = new_mphf.clone();
|
||||
let pass2_err: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
|
||||
let err_cap2 = Arc::clone(&pass2_err);
|
||||
|
||||
let capacity = 2;
|
||||
let throttled_pass2 = throttle(pass2_items.into_iter(), max_open);
|
||||
|
||||
let pipeline2 = Pipeline::new(
|
||||
make_source!(
|
||||
Pass2Data,
|
||||
throttled_pass2.map(|t| {
|
||||
let (col_offset, src_n, unitigs_path, src_layer) = t.item;
|
||||
(col_offset, src_n, unitigs_path, src_layer, t.guard)
|
||||
}),
|
||||
SrcLayer
|
||||
),
|
||||
vec![
|
||||
Stage::Flat(Arc::new(
|
||||
move |data: Pass2Data,
|
||||
push: &PipelineSender<Result<Pass2Data, PipelineError>>,
|
||||
delta: &PipelineSender<isize>| {
|
||||
if let Pass2Data::SrcLayer((
|
||||
col_offset,
|
||||
src_n,
|
||||
unitigs_path,
|
||||
src_layer,
|
||||
_guard,
|
||||
)) = data
|
||||
{
|
||||
// _guard dropped at end of block, releasing the slot.
|
||||
// `src_layer` (MPHF + matrix) was already opened up
|
||||
// front — only the unitigs.bin stream itself is
|
||||
// opened (and throttled) here.
|
||||
let reader = match UnitigFileReader::open_sequential(&unitigs_path) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
*err_cap2.lock().unwrap() = Some(e.to_string());
|
||||
delta.send(-1).ok();
|
||||
return;
|
||||
}
|
||||
};
|
||||
const BATCH: usize = 4096;
|
||||
let mut batch: Vec<CanonicalKmer> = Vec::with_capacity(BATCH);
|
||||
let mut count: isize = 0;
|
||||
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
|
||||
batch.push(kmer);
|
||||
if batch.len() == BATCH {
|
||||
let b = std::mem::replace(&mut batch, Vec::with_capacity(BATCH));
|
||||
push.send(Ok(Pass2Data::RawBatch((
|
||||
col_offset,
|
||||
src_n,
|
||||
Arc::clone(&src_layer),
|
||||
b,
|
||||
))))
|
||||
.ok();
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
push.send(Ok(Pass2Data::RawBatch((
|
||||
col_offset, src_n, src_layer, batch,
|
||||
))))
|
||||
.ok();
|
||||
count += 1;
|
||||
}
|
||||
delta.send(count - 1).ok();
|
||||
}
|
||||
},
|
||||
) as SharedFlatFn<Pass2Data>),
|
||||
make_transform!(
|
||||
Pass2Data,
|
||||
{
|
||||
move |(col_offset, src_n, src_layer, kmers): (
|
||||
usize,
|
||||
usize,
|
||||
Arc<KmerLayer>,
|
||||
Vec<CanonicalKmer>,
|
||||
)|
|
||||
-> Vec<(Option<usize>, usize, Vec<(usize, u32)>)> {
|
||||
// Column-first, one pass for the whole batch: a single
|
||||
// MPHF hash sweep (`hash_batch`) instead of one hash
|
||||
// per kmer, then one read pass per source genome
|
||||
// column (`fill_sub_matrix`) instead of one row read
|
||||
// per kmer — matches the underlying matrix's own
|
||||
// column-major layout.
|
||||
let slots = src_layer.hash_batch(&kmers);
|
||||
let mut cols: Vec<Vec<u32>> = (0..src_n)
|
||||
.map(|_| Vec::with_capacity(kmers.len()))
|
||||
.collect();
|
||||
src_layer.fill_sub_matrix(&slots, &mut cols);
|
||||
|
||||
// Membership against dst, grouped by dst layer
|
||||
// instead of interleaved kmer by kmer — same
|
||||
// grouping `PartitionCache::find_presence_batch`
|
||||
// does in `obikphylo`, via the shared primitive.
|
||||
let (by_dst_layer, not_found) = find_batch_grouped(&dst_layers_t2, &kmers);
|
||||
|
||||
// Genome-major within each layer group (not kmer-major)
|
||||
// so every `(layer, col)` pair's values land together —
|
||||
// one entry per column, not one per `(kmer, genome)`.
|
||||
let mut ops: Vec<(Option<usize>, usize, Vec<(usize, u32)>)> =
|
||||
Vec::with_capacity(src_n * (by_dst_layer.len() + 1));
|
||||
for (dst_layer, hits) in by_dst_layer.into_iter().enumerate() {
|
||||
if hits.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for g in 0..src_n {
|
||||
let col_vals: Vec<(usize, u32)> = hits
|
||||
.iter()
|
||||
.map(|&(idx, slot)| (slot, cols[g][idx]))
|
||||
.collect();
|
||||
ops.push((Some(dst_layer), col_offset + g, col_vals));
|
||||
}
|
||||
}
|
||||
if let (Some(mphf), false) = (&new_mphf_t2, not_found.is_empty()) {
|
||||
let not_found_kmers: Vec<CanonicalKmer> =
|
||||
not_found.iter().map(|&idx| kmers[idx]).collect();
|
||||
let new_slots = mphf.index_batch(¬_found_kmers);
|
||||
for g in 0..src_n {
|
||||
let col_vals: Vec<(usize, u32)> = not_found
|
||||
.iter()
|
||||
.zip(new_slots.iter())
|
||||
.map(|(&idx, &slot)| (slot, cols[g][idx]))
|
||||
.collect();
|
||||
ops.push((None, col_offset + g, col_vals));
|
||||
}
|
||||
}
|
||||
ops
|
||||
}
|
||||
},
|
||||
RawBatch,
|
||||
WriteBatch
|
||||
),
|
||||
],
|
||||
make_sink!(
|
||||
Pass2Data,
|
||||
{
|
||||
move |ops: Vec<(Option<usize>, usize, Vec<(usize, u32)>)>| {
|
||||
for (layer_opt, col, col_vals) in ops {
|
||||
let builder = match layer_opt {
|
||||
Some(l) => &exist_sink[l][col],
|
||||
None => &new_sink[col],
|
||||
};
|
||||
let mut b = builder.lock().unwrap();
|
||||
for (slot, val) in col_vals {
|
||||
b.set_val(slot, val);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
WriteBatch
|
||||
),
|
||||
);
|
||||
|
||||
WorkerPool::new(pipeline2, n_workers, capacity).run();
|
||||
debug!(
|
||||
"partition {i}: pass2 pipeline done in {:.3}s",
|
||||
t_pass2.elapsed().as_secs_f64()
|
||||
);
|
||||
|
||||
if let Some(msg) = Arc::try_unwrap(pass2_err)
|
||||
.unwrap_or_else(|_| panic!("pass2: pass2_err not uniquely owned"))
|
||||
.into_inner()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
{
|
||||
return Err(OKIError::InvalidData {
|
||||
context: "merge pass2",
|
||||
detail: msg,
|
||||
});
|
||||
}
|
||||
|
||||
let t_close = std::time::Instant::now();
|
||||
// ── Close builders ──────────────────────────────────────────────────
|
||||
for (mb, builders) in exist_mbs.into_iter().zip(exist_locked.into_iter()) {
|
||||
for b in builders {
|
||||
Arc::try_unwrap(b)
|
||||
.unwrap_or_else(|_| panic!("pass2: exist_builder not uniquely owned"))
|
||||
.into_inner()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.close()?;
|
||||
}
|
||||
mb.close().map_err(OKIError::Io)?;
|
||||
}
|
||||
|
||||
for b in new_locked {
|
||||
Arc::try_unwrap(b)
|
||||
.unwrap_or_else(|_| panic!("pass2: new_builder not uniquely owned"))
|
||||
.into_inner()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.close()?;
|
||||
}
|
||||
if let Some(mb) = new_mb {
|
||||
mb.close().map_err(OKIError::Io)?;
|
||||
}
|
||||
|
||||
debug!(
|
||||
"partition {i}: builders closed in {:.3}s",
|
||||
t_close.elapsed().as_secs_f64()
|
||||
);
|
||||
|
||||
Ok(n_new)
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
//! Merging a source partition's new layer into a destination partition's
|
||||
//! de Bruijn graph union (pass 1) then column fill (pass 2). Free function,
|
||||
//! not a `KmerIndex` method — `KmerIndex` stays the stateless
|
||||
//! `Index { Partition { Layer } }` data model; this is one of the
|
||||
//! algorithms that reads/writes through it, living here in `obikmerge`
|
||||
//! instead.
|
||||
//!
|
||||
//! Split: this file orchestrates one partition (pass 1, new-layer setup,
|
||||
//! loop over source groups); [`pass2`] holds the per-group column-fill
|
||||
//! pipeline.
|
||||
|
||||
mod pass2;
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use tracing::debug;
|
||||
|
||||
use obicompactvec::{ColBuilder, MatrixBuilder};
|
||||
use obikindex::layer::{IndexMode, KmerLayer, TypedLayer};
|
||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||
use obikindexer::{build_graph, materialize_layer};
|
||||
use obikseq::CanonicalKmer;
|
||||
use obiskio::UnitigFileReader;
|
||||
|
||||
use crate::MergeMode;
|
||||
use pass2::run_pass2;
|
||||
|
||||
// `MatrixBuilder::new`/`resume` take a `presence: bool` — `MergeMode` maps
|
||||
// to it directly (`Presence` → `true`, `Count` → `false`).
|
||||
fn presence(mode: MergeMode) -> bool {
|
||||
mode == MergeMode::Presence
|
||||
}
|
||||
|
||||
// ── merge_partition ────────────────────────────────────────────────────────
|
||||
|
||||
/// Merge `sources` into destination partition `i`.
|
||||
///
|
||||
/// Each entry in `sources` is `(index, n_genomes)` where `n_genomes` is the
|
||||
/// number of genome columns that source contributes. A merged index
|
||||
/// contributes more than one. The total new columns added to the
|
||||
/// destination is `sum(n_genomes)`.
|
||||
///
|
||||
/// `n_dst_genomes` is the number of genome columns already in the
|
||||
/// destination matrices (copied from the bootstrap source before this
|
||||
/// call).
|
||||
pub(crate) fn merge_partition(
|
||||
dst: &KmerIndex,
|
||||
i: usize,
|
||||
sources: &[(&KmerIndex, usize)],
|
||||
mode: MergeMode,
|
||||
n_dst_genomes: usize,
|
||||
block_bits: u8,
|
||||
evidence: &IndexMode,
|
||||
) -> OKIResult<usize> {
|
||||
let dst_partition = dst.partition(i)?;
|
||||
let n_dst_layers = dst_partition.n_layers();
|
||||
|
||||
// First merge in presence mode: materialize implicit single-genome
|
||||
// presence matrices on existing layers (all slots true — every kmer in
|
||||
// those layers belongs to genome_0) so a resumed builder can append
|
||||
// source columns onto them below.
|
||||
if n_dst_genomes == 1 && mode == MergeMode::Presence {
|
||||
for l in 0..n_dst_layers {
|
||||
let layer = dst_partition.layer(l)?.open()?;
|
||||
TypedLayer::<()>::init_presence_matrix(layer.dir(), layer.n())?;
|
||||
}
|
||||
}
|
||||
|
||||
// dst's `0..n_dst_layers` layers, opened once for the whole partition
|
||||
// and shared by pass 1 (filter) and every pass-2 group (membership).
|
||||
// Opened directly, not via `IndexCache`: `obikindexer::build_graph`/
|
||||
// `obipipeline::WorkerPool` are plain-OS-thread pipelines requiring
|
||||
// `Send + Sync + 'static`, and `IndexCache<'_>` always borrows
|
||||
// `&KmerIndex` — only owned `KmerLayer`s can cross that boundary.
|
||||
let dst_layers: Vec<KmerLayer> = (0..n_dst_layers)
|
||||
.map(|l| dst_partition.layer(l)?.open())
|
||||
.collect::<OKIResult<_>>()?;
|
||||
let dst_layers = Arc::new(dst_layers);
|
||||
|
||||
// ── Pass 1: pipeline — parallel file read + dst filter + graph fill ────
|
||||
//
|
||||
// Source : list of unitigs.bin paths (one per source × layer)
|
||||
// Flat : open file, emit Vec<CanonicalKmer> batches (BeeGFS parallel I/O)
|
||||
// Transform: filter via dst_layers — Arc<[KmerLayer]>: Sync
|
||||
// Sink : push new kmers into GraphDeBruijn (single thread, no locks needed)
|
||||
//
|
||||
// Only the unitigs.bin *paths* are needed here: no source layer is
|
||||
// opened (`KmerLayer::new` is the unopened `Empty` variant, whose
|
||||
// `unitigs_path` is valid), so pass 1 holds no source mapping.
|
||||
|
||||
let mut unitig_paths: Vec<PathBuf> = Vec::new();
|
||||
for (src, _) in sources {
|
||||
let partition = src.partition(i)?;
|
||||
for l in 0..partition.n_layers() {
|
||||
let p = KmerLayer::new(&partition, l).unitigs_path();
|
||||
if p.exists() {
|
||||
unitig_paths.push(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let n_src_layers = unitig_paths.len();
|
||||
debug!("partition {i}: de Bruijn graph build start — {n_src_layers} source layer(s)");
|
||||
|
||||
const BATCH: usize = 4096;
|
||||
let n_workers = rayon::current_num_threads().min(16).max(4);
|
||||
// At most 2 files open simultaneously: keeps n_workers-2 workers free
|
||||
// for the Transform stage. Each open file monopolises one worker for the
|
||||
// full duration of its read, so this must stay well below n_workers.
|
||||
let max_open = 2;
|
||||
|
||||
let dst_filter = Arc::clone(&dst_layers);
|
||||
|
||||
let g = build_graph(
|
||||
unitig_paths.into_iter(),
|
||||
move |path: PathBuf, emit: &mut dyn FnMut(Vec<CanonicalKmer>)| -> OKIResult<()> {
|
||||
let reader = UnitigFileReader::open_sequential(&path)?;
|
||||
let mut batch: Vec<CanonicalKmer> = Vec::with_capacity(BATCH);
|
||||
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
|
||||
batch.push(kmer);
|
||||
if batch.len() == BATCH {
|
||||
emit(std::mem::replace(&mut batch, Vec::with_capacity(BATCH)));
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
emit(batch);
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
move |kmer| dst_filter.iter().all(|l| l.find_slot(kmer).is_none()),
|
||||
n_workers,
|
||||
max_open,
|
||||
)?;
|
||||
|
||||
let any_new = g.len() > 0;
|
||||
debug!(
|
||||
"partition {i}: de Bruijn graph done — {} new kmers",
|
||||
g.len()
|
||||
);
|
||||
|
||||
// Build new layer from de Bruijn graph if there are new kmers.
|
||||
let new_layer_idx = n_dst_layers;
|
||||
let new_layer_dir = KmerLayer::new(&dst_partition, new_layer_idx)
|
||||
.dir()
|
||||
.to_path_buf();
|
||||
|
||||
let n_new = if any_new {
|
||||
debug!("partition {i}: unitig traversal start — {} nodes", g.len());
|
||||
let n_nodes = materialize_layer(g, &new_layer_dir, block_bits, evidence)?;
|
||||
debug!("partition {i}: MPHF build done");
|
||||
n_nodes
|
||||
} else {
|
||||
drop(g);
|
||||
0
|
||||
};
|
||||
|
||||
let t_open = std::time::Instant::now();
|
||||
let new_mphf: Option<Arc<obikindex::layer::MphfOnly>> = if any_new {
|
||||
Some(Arc::new(obikindex::layer::MphfOnly::open(&new_layer_dir)?))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
debug!(
|
||||
"partition {i}: MPHF open in {:.3}s",
|
||||
t_open.elapsed().as_secs_f64()
|
||||
);
|
||||
|
||||
let matrix_dir = |layer_dir: &Path| match mode {
|
||||
MergeMode::Presence => layer_dir.join("presence"),
|
||||
MergeMode::Count => layer_dir.join("counts"),
|
||||
};
|
||||
|
||||
// ── New layer matrix: absent (dst-genome) columns only ────────────────
|
||||
// Created and closed here; every source group below resumes it and
|
||||
// appends its own columns, exactly like the existing dst layers.
|
||||
let new_data_dir: Option<PathBuf> = if any_new {
|
||||
let data_dir = matrix_dir(&new_layer_dir);
|
||||
fs::create_dir_all(&data_dir)?;
|
||||
let mut mb = MatrixBuilder::new(presence(mode), n_new, &data_dir).map_err(OKIError::Io)?;
|
||||
for _ in 0..n_dst_genomes {
|
||||
mb.add_absent_col().map_err(OKIError::Io)?;
|
||||
}
|
||||
mb.close().map_err(OKIError::Io)?;
|
||||
Some(data_dir)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let exist_data_dirs: Vec<PathBuf> = dst_layers.iter().map(|l| matrix_dir(l.dir())).collect();
|
||||
|
||||
// ── Pass 2, by consecutive groups of sources ──────────────────────────
|
||||
//
|
||||
// Only one group's source layers and destination columns are alive at a
|
||||
// time, so the number of live mappings per partition no longer grows
|
||||
// with the number of sources. SRC_GROUP matches `max_open`: pass 2
|
||||
// never streams more than `max_open` unitigs.bin files at once, so a
|
||||
// larger group would only keep more layers/columns open without adding
|
||||
// read parallelism.
|
||||
//
|
||||
// Invariant: groups are visited in source order and each appends its
|
||||
// `group_n` columns in source order, so the absolute column of genome
|
||||
// `g` of a source is `n_dst_genomes + col_base + offset_local + g` —
|
||||
// the same column as when every source was processed in one session.
|
||||
//
|
||||
// Memory: the group's columns are built in RAM (`add_mem_col`), at most
|
||||
// `SRC_GROUP × src_n × (L_dst + 1)` live columns per partition, each of
|
||||
// `n_layer / 8` bytes (presence) or `n_layer` bytes + overflow (counts):
|
||||
// about `SRC_GROUP × src_n × N_part × (1/8 | 1)` bytes, `N_part` being
|
||||
// the partition's k-mer count over all dst layers. For mono-genome
|
||||
// sources this stays below pass 1's de Bruijn graph (≥ ~19 B per new
|
||||
// k-mer) unless the new layer is tiny next to the existing ones
|
||||
// (incremental merge into a large index).
|
||||
const SRC_GROUP: usize = 2;
|
||||
let t_pass2 = std::time::Instant::now();
|
||||
let mut col_base = 0usize;
|
||||
for group in sources.chunks(SRC_GROUP) {
|
||||
let group_n: usize = group.iter().map(|(_, n)| *n).sum();
|
||||
|
||||
// This group's source layers, for this partition only. Opened
|
||||
// before any builder so errors propagate via `?` before any worker
|
||||
// thread is spawned. `col_offset` is local to the group.
|
||||
let mut items: Vec<(usize, usize, PathBuf, Arc<KmerLayer>)> = Vec::new();
|
||||
let mut col_offset = 0usize;
|
||||
for (src, src_n) in group {
|
||||
let partition = src.partition(i)?;
|
||||
for l in 0..partition.n_layers() {
|
||||
let layer = Arc::new(partition.layer(l)?.open()?);
|
||||
let unitigs_path = layer.unitigs_path();
|
||||
if unitigs_path.exists() {
|
||||
items.push((col_offset, *src_n, unitigs_path, layer));
|
||||
}
|
||||
}
|
||||
col_offset += src_n;
|
||||
}
|
||||
|
||||
let expected_cols = n_dst_genomes + col_base;
|
||||
let open_group = |data_dir: &Path| -> OKIResult<(MatrixBuilder, Vec<ColBuilder>)> {
|
||||
let mut mb = MatrixBuilder::resume(presence(mode), data_dir).map_err(OKIError::Io)?;
|
||||
if mb.n_cols() != expected_cols {
|
||||
return Err(OKIError::InvalidData {
|
||||
context: "merge pass2",
|
||||
detail: format!(
|
||||
"{}: {} columns, expected {expected_cols}",
|
||||
data_dir.display(),
|
||||
mb.n_cols()
|
||||
),
|
||||
});
|
||||
}
|
||||
// In-memory columns: no sparse file / `MmapMut` on the write
|
||||
// path, so a block the FS cannot allocate is an `io::Error` at
|
||||
// `close`, not a SIGBUS (see `MatrixBuilder::add_mem_col`).
|
||||
let cols = (0..group_n).map(|_| mb.add_mem_col()).collect();
|
||||
Ok((mb, cols))
|
||||
};
|
||||
|
||||
let (exist_mbs, exist_cols): (Vec<MatrixBuilder>, Vec<Vec<ColBuilder>>) = exist_data_dirs
|
||||
.iter()
|
||||
.map(|d| open_group(d))
|
||||
.collect::<OKIResult<Vec<_>>>()?
|
||||
.into_iter()
|
||||
.unzip();
|
||||
let (new_mb, new_cols) = match &new_data_dir {
|
||||
Some(d) => {
|
||||
let (mb, cols) = open_group(d)?;
|
||||
(Some(mb), cols)
|
||||
}
|
||||
None => (None, vec![]),
|
||||
};
|
||||
|
||||
// `items` (and with it this group's source layers) is consumed by
|
||||
// the pipeline and dropped when it returns.
|
||||
let (exist_cols, new_cols) = run_pass2(
|
||||
items,
|
||||
&dst_layers,
|
||||
&new_mphf,
|
||||
exist_cols,
|
||||
new_cols,
|
||||
n_workers,
|
||||
max_open,
|
||||
)?;
|
||||
|
||||
for (mb, cols) in exist_mbs.into_iter().zip(exist_cols) {
|
||||
for c in cols {
|
||||
c.close()?;
|
||||
}
|
||||
mb.close().map_err(OKIError::Io)?;
|
||||
}
|
||||
for c in new_cols {
|
||||
c.close()?;
|
||||
}
|
||||
if let Some(mb) = new_mb {
|
||||
mb.close().map_err(OKIError::Io)?;
|
||||
}
|
||||
|
||||
col_base += group_n;
|
||||
}
|
||||
debug!(
|
||||
"partition {i}: pass2 done in {:.3}s",
|
||||
t_pass2.elapsed().as_secs_f64()
|
||||
);
|
||||
|
||||
Ok(n_new)
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
//! Pass 2 of a partition merge, for one group of sources: stream each
|
||||
//! source layer's k-mers, read their values from the source matrix, locate
|
||||
//! them in dst (existing layers, else the new layer's MPHF) and write them
|
||||
//! into the group's destination columns.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use obipipeline::{
|
||||
Pipeline, PipelineError, PipelineSender, SharedFlatFn, Stage, ThrottleGuard, WorkerPool,
|
||||
make_sink, make_source, make_transform, throttle,
|
||||
};
|
||||
|
||||
use obicompactvec::ColBuilder;
|
||||
use obikindex::layer::KmerLayer;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
use obikseq::CanonicalKmer;
|
||||
use obiskio::UnitigFileReader;
|
||||
|
||||
// ── dst cross-layer lookup ──────────────────────────────────────────────────
|
||||
|
||||
/// Batch membership lookup across `dst_layers` — first layer that carries
|
||||
/// each kmer wins. Groups hits by layer index (as `(kmer batch index,
|
||||
/// slot)` pairs) rather than returning them in query order, so pass 2 can
|
||||
/// follow with one column-major write per `(layer, column)` instead of
|
||||
/// interleaving across layers kmer by kmer. The kmers with no hit in any
|
||||
/// layer come back as a separate index list, for the new-layer MPHF batch
|
||||
/// lookup on a miss.
|
||||
fn find_batch_grouped(
|
||||
layers: &[KmerLayer],
|
||||
kmers: &[CanonicalKmer],
|
||||
) -> (Vec<Vec<(usize, usize)>>, Vec<usize>) {
|
||||
let mut by_layer: Vec<Vec<(usize, usize)>> = vec![Vec::new(); layers.len()];
|
||||
let mut not_found: Vec<usize> = Vec::new();
|
||||
for (idx, &kmer) in kmers.iter().enumerate() {
|
||||
match layers
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find_map(|(li, l)| l.find_slot(kmer).map(|slot| (li, slot)))
|
||||
{
|
||||
Some((li, slot)) => by_layer[li].push((idx, slot)),
|
||||
None => not_found.push(idx),
|
||||
}
|
||||
}
|
||||
(by_layer, not_found)
|
||||
}
|
||||
|
||||
|
||||
// ── per-group pipeline ─────────────────────────────────────────────────────
|
||||
|
||||
/// Pass 2 for one group of sources: stream each source layer's unitigs,
|
||||
/// look its k-mers up in the source matrix and in dst (existing layers,
|
||||
/// then the new layer's MPHF), and write the values into the group's
|
||||
/// column builders. `items` carry group-local column offsets; the builders
|
||||
/// are handed back (still open) for the caller to close.
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub(super) fn run_pass2(
|
||||
items: Vec<(usize, usize, PathBuf, Arc<KmerLayer>)>,
|
||||
dst_layers: &Arc<Vec<KmerLayer>>,
|
||||
new_mphf: &Option<Arc<obikindex::layer::MphfOnly>>,
|
||||
exist_builders: Vec<Vec<ColBuilder>>,
|
||||
new_builders: Vec<ColBuilder>,
|
||||
n_workers: usize,
|
||||
max_open: usize,
|
||||
) -> OKIResult<(Vec<Vec<ColBuilder>>, Vec<ColBuilder>)> {
|
||||
enum Pass2Data {
|
||||
SrcLayer((usize, usize, PathBuf, Arc<KmerLayer>, ThrottleGuard)),
|
||||
RawBatch((usize, usize, Arc<KmerLayer>, Vec<CanonicalKmer>)),
|
||||
/// One entry per `(dst layer, column)` touched by this batch, each
|
||||
/// carrying every `(slot, value)` pair for that column together —
|
||||
/// column-major, so the sink below locks that column's `ColBuilder`
|
||||
/// once per batch and writes the whole run, instead of once per
|
||||
/// `(kmer, genome)` pair.
|
||||
WriteBatch(Vec<(Option<usize>, usize, Vec<(usize, u32)>)>),
|
||||
}
|
||||
|
||||
let exist_locked: Vec<Vec<Arc<Mutex<ColBuilder>>>> = exist_builders
|
||||
.into_iter()
|
||||
.map(|layer| layer.into_iter().map(|b| Arc::new(Mutex::new(b))).collect())
|
||||
.collect();
|
||||
let new_locked: Vec<Arc<Mutex<ColBuilder>>> = new_builders
|
||||
.into_iter()
|
||||
.map(|b| Arc::new(Mutex::new(b)))
|
||||
.collect();
|
||||
let exist_sink: Vec<Vec<Arc<Mutex<ColBuilder>>>> = exist_locked
|
||||
.iter()
|
||||
.map(|layer| layer.iter().map(Arc::clone).collect())
|
||||
.collect();
|
||||
let new_sink: Vec<Arc<Mutex<ColBuilder>>> = new_locked.iter().map(Arc::clone).collect();
|
||||
let dst_layers_t2 = Arc::clone(dst_layers);
|
||||
let new_mphf_t2 = new_mphf.clone();
|
||||
let pass2_err: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
|
||||
let err_cap2 = Arc::clone(&pass2_err);
|
||||
|
||||
let capacity = 2;
|
||||
let throttled_pass2 = throttle(items.into_iter(), max_open);
|
||||
|
||||
let pipeline2 = Pipeline::new(
|
||||
make_source!(
|
||||
Pass2Data,
|
||||
throttled_pass2.map(|t| {
|
||||
let (col_offset, src_n, unitigs_path, src_layer) = t.item;
|
||||
(col_offset, src_n, unitigs_path, src_layer, t.guard)
|
||||
}),
|
||||
SrcLayer
|
||||
),
|
||||
vec![
|
||||
Stage::Flat(Arc::new(
|
||||
move |data: Pass2Data,
|
||||
push: &PipelineSender<Result<Pass2Data, PipelineError>>,
|
||||
delta: &PipelineSender<isize>| {
|
||||
if let Pass2Data::SrcLayer((
|
||||
col_offset,
|
||||
src_n,
|
||||
unitigs_path,
|
||||
src_layer,
|
||||
_guard,
|
||||
)) = data
|
||||
{
|
||||
// _guard dropped at end of block, releasing the slot.
|
||||
// `src_layer` (MPHF + matrix) was already opened by
|
||||
// the caller — only the unitigs.bin stream itself is
|
||||
// opened (and throttled) here.
|
||||
let reader = match UnitigFileReader::open_sequential(&unitigs_path) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
*err_cap2.lock().unwrap() = Some(e.to_string());
|
||||
delta.send(-1).ok();
|
||||
return;
|
||||
}
|
||||
};
|
||||
const BATCH: usize = 4096;
|
||||
let mut batch: Vec<CanonicalKmer> = Vec::with_capacity(BATCH);
|
||||
let mut count: isize = 0;
|
||||
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
|
||||
batch.push(kmer);
|
||||
if batch.len() == BATCH {
|
||||
let b = std::mem::replace(&mut batch, Vec::with_capacity(BATCH));
|
||||
push.send(Ok(Pass2Data::RawBatch((
|
||||
col_offset,
|
||||
src_n,
|
||||
Arc::clone(&src_layer),
|
||||
b,
|
||||
))))
|
||||
.ok();
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
push.send(Ok(Pass2Data::RawBatch((
|
||||
col_offset, src_n, src_layer, batch,
|
||||
))))
|
||||
.ok();
|
||||
count += 1;
|
||||
}
|
||||
delta.send(count - 1).ok();
|
||||
}
|
||||
},
|
||||
) as SharedFlatFn<Pass2Data>),
|
||||
make_transform!(
|
||||
Pass2Data,
|
||||
{
|
||||
move |(col_offset, src_n, src_layer, kmers): (
|
||||
usize,
|
||||
usize,
|
||||
Arc<KmerLayer>,
|
||||
Vec<CanonicalKmer>,
|
||||
)|
|
||||
-> Vec<(Option<usize>, usize, Vec<(usize, u32)>)> {
|
||||
// Column-first, one pass for the whole batch: a single
|
||||
// MPHF hash sweep (`hash_batch`) instead of one hash
|
||||
// per kmer, then one read pass per source genome
|
||||
// column (`fill_sub_matrix`) instead of one row read
|
||||
// per kmer — matches the underlying matrix's own
|
||||
// column-major layout.
|
||||
let slots = src_layer.hash_batch(&kmers);
|
||||
let mut cols: Vec<Vec<u32>> = (0..src_n)
|
||||
.map(|_| Vec::with_capacity(kmers.len()))
|
||||
.collect();
|
||||
src_layer.fill_sub_matrix(&slots, &mut cols);
|
||||
|
||||
// Membership against dst, grouped by dst layer
|
||||
// instead of interleaved kmer by kmer — same
|
||||
// grouping `PartitionCache::find_presence_batch`
|
||||
// does in `obikphylo`, via the shared primitive.
|
||||
let (by_dst_layer, not_found) = find_batch_grouped(&dst_layers_t2, &kmers);
|
||||
|
||||
// Genome-major within each layer group (not kmer-major)
|
||||
// so every `(layer, col)` pair's values land together —
|
||||
// one entry per column, not one per `(kmer, genome)`.
|
||||
let mut ops: Vec<(Option<usize>, usize, Vec<(usize, u32)>)> =
|
||||
Vec::with_capacity(src_n * (by_dst_layer.len() + 1));
|
||||
for (dst_layer, hits) in by_dst_layer.into_iter().enumerate() {
|
||||
if hits.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for g in 0..src_n {
|
||||
let col_vals: Vec<(usize, u32)> = hits
|
||||
.iter()
|
||||
.map(|&(idx, slot)| (slot, cols[g][idx]))
|
||||
.collect();
|
||||
ops.push((Some(dst_layer), col_offset + g, col_vals));
|
||||
}
|
||||
}
|
||||
if let (Some(mphf), false) = (&new_mphf_t2, not_found.is_empty()) {
|
||||
let not_found_kmers: Vec<CanonicalKmer> =
|
||||
not_found.iter().map(|&idx| kmers[idx]).collect();
|
||||
let new_slots = mphf.index_batch(¬_found_kmers);
|
||||
for g in 0..src_n {
|
||||
let col_vals: Vec<(usize, u32)> = not_found
|
||||
.iter()
|
||||
.zip(new_slots.iter())
|
||||
.map(|(&idx, &slot)| (slot, cols[g][idx]))
|
||||
.collect();
|
||||
ops.push((None, col_offset + g, col_vals));
|
||||
}
|
||||
}
|
||||
ops
|
||||
}
|
||||
},
|
||||
RawBatch,
|
||||
WriteBatch
|
||||
),
|
||||
],
|
||||
make_sink!(
|
||||
Pass2Data,
|
||||
{
|
||||
move |ops: Vec<(Option<usize>, usize, Vec<(usize, u32)>)>| {
|
||||
for (layer_opt, col, col_vals) in ops {
|
||||
let builder = match layer_opt {
|
||||
Some(l) => &exist_sink[l][col],
|
||||
None => &new_sink[col],
|
||||
};
|
||||
let mut b = builder.lock().unwrap();
|
||||
for (slot, val) in col_vals {
|
||||
b.set_val(slot, val);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
WriteBatch
|
||||
),
|
||||
);
|
||||
|
||||
WorkerPool::new(pipeline2, n_workers, capacity).run();
|
||||
|
||||
if let Some(msg) = Arc::try_unwrap(pass2_err)
|
||||
.unwrap_or_else(|_| panic!("pass2: pass2_err not uniquely owned"))
|
||||
.into_inner()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
{
|
||||
return Err(OKIError::InvalidData {
|
||||
context: "merge pass2",
|
||||
detail: msg,
|
||||
});
|
||||
}
|
||||
|
||||
let unlock = |b: Arc<Mutex<ColBuilder>>| -> ColBuilder {
|
||||
Arc::try_unwrap(b)
|
||||
.unwrap_or_else(|_| panic!("pass2: column builder not uniquely owned"))
|
||||
.into_inner()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
};
|
||||
let exist = exist_locked
|
||||
.into_iter()
|
||||
.map(|layer| layer.into_iter().map(unlock).collect())
|
||||
.collect();
|
||||
let new = new_locked.into_iter().map(unlock).collect();
|
||||
Ok((exist, new))
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user