Introduce `MemBitVecBuilder` and `MemCompactIntVecBuilder` to construct columns in RAM before writing, reducing the number of open file descriptors during merge operations. Refactor `partition_merge.rs` to use these new builders via `add_mem_col`, decoupling live memory mappings from source counts to prevent `SIGBUS` errors. - Add `mem_builder` module with `MemBitVecBuilder` and `MemCompactIntVecBuilder` structs for in-memory column construction and PCIV file writing. - Extend `MatrixBuilder` with `add_mem_col` and `n_cols` methods, updating `ColBuilder` enum to support `MemBit` and `MemInt` variants. - Refactor `partition_merge.rs` to use `add_mem_col` instead of `add_col`, introducing `run_pass2` for batch processing and optimizing destination layer handling. - Update `format.rs` with `pciv_index_layout`, `write_pciv_header`, and `write_pciv_tail` functions to manage PCIV index layout and header/tail sections. - Modify `mmap_file.rs` to use `write_std_header` and introduce `write_built` for standardized file writing and flushing logic. - Make `MAGIC` constant public in `bitvec.rs` and update documentation for memory mapping alignment and structure. - Add test modules `tests/matrix_builder.rs` and `tests/mem_builder.rs` to verify column counts and byte-identical output of new builders against persistent versions. - Update `mkdocs.yml` navigation to include "Merge mmap audit (2026-11)" documentation entry. - Add implementation audit documentation in `DevDocMD/implementation/merge_mmap_audit.md` detailing memory mapping failures and refactoring plans. - Add merge testing scripts `scripts/smoke_test_merge.sh` and `scripts/bench_merge_oracle.sh` for validation and benchmarking. - Bump `obikmer` package version from 1.4.2 to 1.4.3 in `Cargo.toml`.
305 lines
12 KiB
Rust
305 lines
12 KiB
Rust
//! Merging a source partition's new layer into a destination partition's
|
|
//! de Bruijn graph union (pass 1) then column fill (pass 2). Free function,
|
|
//! not a `KmerIndex` method — `KmerIndex` stays the stateless
|
|
//! `Index { Partition { Layer } }` data model; this is one of the
|
|
//! algorithms that reads/writes through it, living here in `obikmerge`
|
|
//! instead.
|
|
//!
|
|
//! Split: this file orchestrates one partition (pass 1, new-layer setup,
|
|
//! loop over source groups); [`pass2`] holds the per-group column-fill
|
|
//! pipeline.
|
|
|
|
mod pass2;
|
|
|
|
use std::fs;
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::Arc;
|
|
|
|
use tracing::debug;
|
|
|
|
use obicompactvec::{ColBuilder, MatrixBuilder};
|
|
use obikindex::layer::{IndexMode, KmerLayer, TypedLayer};
|
|
use obikindex::{KmerIndex, OKIError, OKIResult};
|
|
use obikindexer::{build_graph, materialize_layer};
|
|
use obikseq::CanonicalKmer;
|
|
use obiskio::UnitigFileReader;
|
|
|
|
use crate::MergeMode;
|
|
use pass2::run_pass2;
|
|
|
|
// `MatrixBuilder::new`/`resume` take a `presence: bool` — `MergeMode` maps
|
|
// to it directly (`Presence` → `true`, `Count` → `false`).
|
|
fn presence(mode: MergeMode) -> bool {
|
|
mode == MergeMode::Presence
|
|
}
|
|
|
|
// ── merge_partition ────────────────────────────────────────────────────────
|
|
|
|
/// Merge `sources` into destination partition `i`.
|
|
///
|
|
/// Each entry in `sources` is `(index, n_genomes)` where `n_genomes` is the
|
|
/// number of genome columns that source contributes. A merged index
|
|
/// contributes more than one. The total new columns added to the
|
|
/// destination is `sum(n_genomes)`.
|
|
///
|
|
/// `n_dst_genomes` is the number of genome columns already in the
|
|
/// destination matrices (copied from the bootstrap source before this
|
|
/// call).
|
|
pub(crate) fn merge_partition(
|
|
dst: &KmerIndex,
|
|
i: usize,
|
|
sources: &[(&KmerIndex, usize)],
|
|
mode: MergeMode,
|
|
n_dst_genomes: usize,
|
|
block_bits: u8,
|
|
evidence: &IndexMode,
|
|
) -> OKIResult<usize> {
|
|
let dst_partition = dst.partition(i)?;
|
|
let n_dst_layers = dst_partition.n_layers();
|
|
|
|
// First merge in presence mode: materialize implicit single-genome
|
|
// presence matrices on existing layers (all slots true — every kmer in
|
|
// those layers belongs to genome_0) so a resumed builder can append
|
|
// source columns onto them below.
|
|
if n_dst_genomes == 1 && mode == MergeMode::Presence {
|
|
for l in 0..n_dst_layers {
|
|
let layer = dst_partition.layer(l)?.open()?;
|
|
TypedLayer::<()>::init_presence_matrix(layer.dir(), layer.n())?;
|
|
}
|
|
}
|
|
|
|
// dst's `0..n_dst_layers` layers, opened once for the whole partition
|
|
// and shared by pass 1 (filter) and every pass-2 group (membership).
|
|
// Opened directly, not via `IndexCache`: `obikindexer::build_graph`/
|
|
// `obipipeline::WorkerPool` are plain-OS-thread pipelines requiring
|
|
// `Send + Sync + 'static`, and `IndexCache<'_>` always borrows
|
|
// `&KmerIndex` — only owned `KmerLayer`s can cross that boundary.
|
|
let dst_layers: Vec<KmerLayer> = (0..n_dst_layers)
|
|
.map(|l| dst_partition.layer(l)?.open())
|
|
.collect::<OKIResult<_>>()?;
|
|
let dst_layers = Arc::new(dst_layers);
|
|
|
|
// ── Pass 1: pipeline — parallel file read + dst filter + graph fill ────
|
|
//
|
|
// Source : list of unitigs.bin paths (one per source × layer)
|
|
// Flat : open file, emit Vec<CanonicalKmer> batches (BeeGFS parallel I/O)
|
|
// Transform: filter via dst_layers — Arc<[KmerLayer]>: Sync
|
|
// Sink : push new kmers into GraphDeBruijn (single thread, no locks needed)
|
|
//
|
|
// Only the unitigs.bin *paths* are needed here: no source layer is
|
|
// opened (`KmerLayer::new` is the unopened `Empty` variant, whose
|
|
// `unitigs_path` is valid), so pass 1 holds no source mapping.
|
|
|
|
let mut unitig_paths: Vec<PathBuf> = Vec::new();
|
|
for (src, _) in sources {
|
|
let partition = src.partition(i)?;
|
|
for l in 0..partition.n_layers() {
|
|
let p = KmerLayer::new(&partition, l).unitigs_path();
|
|
if p.exists() {
|
|
unitig_paths.push(p);
|
|
}
|
|
}
|
|
}
|
|
|
|
let n_src_layers = unitig_paths.len();
|
|
debug!("partition {i}: de Bruijn graph build start — {n_src_layers} source layer(s)");
|
|
|
|
const BATCH: usize = 4096;
|
|
let n_workers = rayon::current_num_threads().min(16).max(4);
|
|
// At most 2 files open simultaneously: keeps n_workers-2 workers free
|
|
// for the Transform stage. Each open file monopolises one worker for the
|
|
// full duration of its read, so this must stay well below n_workers.
|
|
let max_open = 2;
|
|
|
|
let dst_filter = Arc::clone(&dst_layers);
|
|
|
|
let g = build_graph(
|
|
unitig_paths.into_iter(),
|
|
move |path: PathBuf, emit: &mut dyn FnMut(Vec<CanonicalKmer>)| -> OKIResult<()> {
|
|
let reader = UnitigFileReader::open_sequential(&path)?;
|
|
let mut batch: Vec<CanonicalKmer> = Vec::with_capacity(BATCH);
|
|
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
|
|
batch.push(kmer);
|
|
if batch.len() == BATCH {
|
|
emit(std::mem::replace(&mut batch, Vec::with_capacity(BATCH)));
|
|
}
|
|
}
|
|
if !batch.is_empty() {
|
|
emit(batch);
|
|
}
|
|
Ok(())
|
|
},
|
|
move |kmer| dst_filter.iter().all(|l| l.find_slot(kmer).is_none()),
|
|
n_workers,
|
|
max_open,
|
|
)?;
|
|
|
|
let any_new = g.len() > 0;
|
|
debug!(
|
|
"partition {i}: de Bruijn graph done — {} new kmers",
|
|
g.len()
|
|
);
|
|
|
|
// Build new layer from de Bruijn graph if there are new kmers.
|
|
let new_layer_idx = n_dst_layers;
|
|
let new_layer_dir = KmerLayer::new(&dst_partition, new_layer_idx)
|
|
.dir()
|
|
.to_path_buf();
|
|
|
|
let n_new = if any_new {
|
|
debug!("partition {i}: unitig traversal start — {} nodes", g.len());
|
|
let n_nodes = materialize_layer(g, &new_layer_dir, block_bits, evidence)?;
|
|
debug!("partition {i}: MPHF build done");
|
|
n_nodes
|
|
} else {
|
|
drop(g);
|
|
0
|
|
};
|
|
|
|
let t_open = std::time::Instant::now();
|
|
let new_mphf: Option<Arc<obikindex::layer::MphfOnly>> = if any_new {
|
|
Some(Arc::new(obikindex::layer::MphfOnly::open(&new_layer_dir)?))
|
|
} else {
|
|
None
|
|
};
|
|
debug!(
|
|
"partition {i}: MPHF open in {:.3}s",
|
|
t_open.elapsed().as_secs_f64()
|
|
);
|
|
|
|
let matrix_dir = |layer_dir: &Path| match mode {
|
|
MergeMode::Presence => layer_dir.join("presence"),
|
|
MergeMode::Count => layer_dir.join("counts"),
|
|
};
|
|
|
|
// ── New layer matrix: absent (dst-genome) columns only ────────────────
|
|
// Created and closed here; every source group below resumes it and
|
|
// appends its own columns, exactly like the existing dst layers.
|
|
let new_data_dir: Option<PathBuf> = if any_new {
|
|
let data_dir = matrix_dir(&new_layer_dir);
|
|
fs::create_dir_all(&data_dir)?;
|
|
let mut mb = MatrixBuilder::new(presence(mode), n_new, &data_dir).map_err(OKIError::Io)?;
|
|
for _ in 0..n_dst_genomes {
|
|
mb.add_absent_col().map_err(OKIError::Io)?;
|
|
}
|
|
mb.close().map_err(OKIError::Io)?;
|
|
Some(data_dir)
|
|
} else {
|
|
None
|
|
};
|
|
let exist_data_dirs: Vec<PathBuf> = dst_layers.iter().map(|l| matrix_dir(l.dir())).collect();
|
|
|
|
// ── Pass 2, by consecutive groups of sources ──────────────────────────
|
|
//
|
|
// Only one group's source layers and destination columns are alive at a
|
|
// time, so the number of live mappings per partition no longer grows
|
|
// with the number of sources. SRC_GROUP matches `max_open`: pass 2
|
|
// never streams more than `max_open` unitigs.bin files at once, so a
|
|
// larger group would only keep more layers/columns open without adding
|
|
// read parallelism.
|
|
//
|
|
// Invariant: groups are visited in source order and each appends its
|
|
// `group_n` columns in source order, so the absolute column of genome
|
|
// `g` of a source is `n_dst_genomes + col_base + offset_local + g` —
|
|
// the same column as when every source was processed in one session.
|
|
//
|
|
// Memory: the group's columns are built in RAM (`add_mem_col`), at most
|
|
// `SRC_GROUP × src_n × (L_dst + 1)` live columns per partition, each of
|
|
// `n_layer / 8` bytes (presence) or `n_layer` bytes + overflow (counts):
|
|
// about `SRC_GROUP × src_n × N_part × (1/8 | 1)` bytes, `N_part` being
|
|
// the partition's k-mer count over all dst layers. For mono-genome
|
|
// sources this stays below pass 1's de Bruijn graph (≥ ~19 B per new
|
|
// k-mer) unless the new layer is tiny next to the existing ones
|
|
// (incremental merge into a large index).
|
|
const SRC_GROUP: usize = 2;
|
|
let t_pass2 = std::time::Instant::now();
|
|
let mut col_base = 0usize;
|
|
for group in sources.chunks(SRC_GROUP) {
|
|
let group_n: usize = group.iter().map(|(_, n)| *n).sum();
|
|
|
|
// This group's source layers, for this partition only. Opened
|
|
// before any builder so errors propagate via `?` before any worker
|
|
// thread is spawned. `col_offset` is local to the group.
|
|
let mut items: Vec<(usize, usize, PathBuf, Arc<KmerLayer>)> = Vec::new();
|
|
let mut col_offset = 0usize;
|
|
for (src, src_n) in group {
|
|
let partition = src.partition(i)?;
|
|
for l in 0..partition.n_layers() {
|
|
let layer = Arc::new(partition.layer(l)?.open()?);
|
|
let unitigs_path = layer.unitigs_path();
|
|
if unitigs_path.exists() {
|
|
items.push((col_offset, *src_n, unitigs_path, layer));
|
|
}
|
|
}
|
|
col_offset += src_n;
|
|
}
|
|
|
|
let expected_cols = n_dst_genomes + col_base;
|
|
let open_group = |data_dir: &Path| -> OKIResult<(MatrixBuilder, Vec<ColBuilder>)> {
|
|
let mut mb = MatrixBuilder::resume(presence(mode), data_dir).map_err(OKIError::Io)?;
|
|
if mb.n_cols() != expected_cols {
|
|
return Err(OKIError::InvalidData {
|
|
context: "merge pass2",
|
|
detail: format!(
|
|
"{}: {} columns, expected {expected_cols}",
|
|
data_dir.display(),
|
|
mb.n_cols()
|
|
),
|
|
});
|
|
}
|
|
// In-memory columns: no sparse file / `MmapMut` on the write
|
|
// path, so a block the FS cannot allocate is an `io::Error` at
|
|
// `close`, not a SIGBUS (see `MatrixBuilder::add_mem_col`).
|
|
let cols = (0..group_n).map(|_| mb.add_mem_col()).collect();
|
|
Ok((mb, cols))
|
|
};
|
|
|
|
let (exist_mbs, exist_cols): (Vec<MatrixBuilder>, Vec<Vec<ColBuilder>>) = exist_data_dirs
|
|
.iter()
|
|
.map(|d| open_group(d))
|
|
.collect::<OKIResult<Vec<_>>>()?
|
|
.into_iter()
|
|
.unzip();
|
|
let (new_mb, new_cols) = match &new_data_dir {
|
|
Some(d) => {
|
|
let (mb, cols) = open_group(d)?;
|
|
(Some(mb), cols)
|
|
}
|
|
None => (None, vec![]),
|
|
};
|
|
|
|
// `items` (and with it this group's source layers) is consumed by
|
|
// the pipeline and dropped when it returns.
|
|
let (exist_cols, new_cols) = run_pass2(
|
|
items,
|
|
&dst_layers,
|
|
&new_mphf,
|
|
exist_cols,
|
|
new_cols,
|
|
n_workers,
|
|
max_open,
|
|
)?;
|
|
|
|
for (mb, cols) in exist_mbs.into_iter().zip(exist_cols) {
|
|
for c in cols {
|
|
c.close()?;
|
|
}
|
|
mb.close().map_err(OKIError::Io)?;
|
|
}
|
|
for c in new_cols {
|
|
c.close()?;
|
|
}
|
|
if let Some(mb) = new_mb {
|
|
mb.close().map_err(OKIError::Io)?;
|
|
}
|
|
|
|
col_base += group_n;
|
|
}
|
|
debug!(
|
|
"partition {i}: pass2 done in {:.3}s",
|
|
t_pass2.elapsed().as_secs_f64()
|
|
);
|
|
|
|
Ok(n_new)
|
|
}
|