Files
obikmer/src/obikmerge/src/partition_merge/mod.rs
T
Eric Coissac 1f199c69db feat: implement in-memory column builders and refactor merge partitioning
Introduce `MemBitVecBuilder` and `MemCompactIntVecBuilder` to construct columns in RAM before writing, reducing the number of open file descriptors during merge operations. Refactor `partition_merge.rs` to use these new builders via `add_mem_col`, decoupling live memory mappings from source counts to prevent `SIGBUS` errors.

- Add `mem_builder` module with `MemBitVecBuilder` and `MemCompactIntVecBuilder` structs for in-memory column construction and PCIV file writing.
- Extend `MatrixBuilder` with `add_mem_col` and `n_cols` methods, updating `ColBuilder` enum to support `MemBit` and `MemInt` variants.
- Refactor `partition_merge.rs` to use `add_mem_col` instead of `add_col`, introducing `run_pass2` for batch processing and optimizing destination layer handling.
- Update `format.rs` with `pciv_index_layout`, `write_pciv_header`, and `write_pciv_tail` functions to manage PCIV index layout and header/tail sections.
- Modify `mmap_file.rs` to use `write_std_header` and introduce `write_built` for standardized file writing and flushing logic.
- Make `MAGIC` constant public in `bitvec.rs` and update documentation for memory mapping alignment and structure.
- Add test modules `tests/matrix_builder.rs` and `tests/mem_builder.rs` to verify column counts and byte-identical output of new builders against persistent versions.
- Update `mkdocs.yml` navigation to include "Merge mmap audit (2026-11)" documentation entry.
- Add implementation audit documentation in `DevDocMD/implementation/merge_mmap_audit.md` detailing memory mapping failures and refactoring plans.
- Add merge testing scripts `scripts/smoke_test_merge.sh` and `scripts/bench_merge_oracle.sh` for validation and benchmarking.
- Bump `obikmer` package version from 1.4.2 to 1.4.3 in `Cargo.toml`.
2026-11-17 15:16:41 +01:00

305 lines
12 KiB
Rust

//! Merging a source partition's new layer into a destination partition's
//! de Bruijn graph union (pass 1) then column fill (pass 2). Free function,
//! not a `KmerIndex` method — `KmerIndex` stays the stateless
//! `Index { Partition { Layer } }` data model; this is one of the
//! algorithms that reads/writes through it, living here in `obikmerge`
//! instead.
//!
//! Split: this file orchestrates one partition (pass 1, new-layer setup,
//! loop over source groups); [`pass2`] holds the per-group column-fill
//! pipeline.
mod pass2;
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tracing::debug;
use obicompactvec::{ColBuilder, MatrixBuilder};
use obikindex::layer::{IndexMode, KmerLayer, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult};
use obikindexer::{build_graph, materialize_layer};
use obikseq::CanonicalKmer;
use obiskio::UnitigFileReader;
use crate::MergeMode;
use pass2::run_pass2;
// `MatrixBuilder::new`/`resume` take a `presence: bool` — `MergeMode` maps
// to it directly (`Presence` → `true`, `Count` → `false`).
fn presence(mode: MergeMode) -> bool {
mode == MergeMode::Presence
}
// ── merge_partition ────────────────────────────────────────────────────────
/// Merge `sources` into destination partition `i`.
///
/// Each entry in `sources` is `(index, n_genomes)` where `n_genomes` is the
/// number of genome columns that source contributes. A merged index
/// contributes more than one. The total new columns added to the
/// destination is `sum(n_genomes)`.
///
/// `n_dst_genomes` is the number of genome columns already in the
/// destination matrices (copied from the bootstrap source before this
/// call).
pub(crate) fn merge_partition(
dst: &KmerIndex,
i: usize,
sources: &[(&KmerIndex, usize)],
mode: MergeMode,
n_dst_genomes: usize,
block_bits: u8,
evidence: &IndexMode,
) -> OKIResult<usize> {
let dst_partition = dst.partition(i)?;
let n_dst_layers = dst_partition.n_layers();
// First merge in presence mode: materialize implicit single-genome
// presence matrices on existing layers (all slots true — every kmer in
// those layers belongs to genome_0) so a resumed builder can append
// source columns onto them below.
if n_dst_genomes == 1 && mode == MergeMode::Presence {
for l in 0..n_dst_layers {
let layer = dst_partition.layer(l)?.open()?;
TypedLayer::<()>::init_presence_matrix(layer.dir(), layer.n())?;
}
}
// dst's `0..n_dst_layers` layers, opened once for the whole partition
// and shared by pass 1 (filter) and every pass-2 group (membership).
// Opened directly, not via `IndexCache`: `obikindexer::build_graph`/
// `obipipeline::WorkerPool` are plain-OS-thread pipelines requiring
// `Send + Sync + 'static`, and `IndexCache<'_>` always borrows
// `&KmerIndex` — only owned `KmerLayer`s can cross that boundary.
let dst_layers: Vec<KmerLayer> = (0..n_dst_layers)
.map(|l| dst_partition.layer(l)?.open())
.collect::<OKIResult<_>>()?;
let dst_layers = Arc::new(dst_layers);
// ── Pass 1: pipeline — parallel file read + dst filter + graph fill ────
//
// Source : list of unitigs.bin paths (one per source × layer)
// Flat : open file, emit Vec<CanonicalKmer> batches (BeeGFS parallel I/O)
// Transform: filter via dst_layers — Arc<[KmerLayer]>: Sync
// Sink : push new kmers into GraphDeBruijn (single thread, no locks needed)
//
// Only the unitigs.bin *paths* are needed here: no source layer is
// opened (`KmerLayer::new` is the unopened `Empty` variant, whose
// `unitigs_path` is valid), so pass 1 holds no source mapping.
let mut unitig_paths: Vec<PathBuf> = Vec::new();
for (src, _) in sources {
let partition = src.partition(i)?;
for l in 0..partition.n_layers() {
let p = KmerLayer::new(&partition, l).unitigs_path();
if p.exists() {
unitig_paths.push(p);
}
}
}
let n_src_layers = unitig_paths.len();
debug!("partition {i}: de Bruijn graph build start — {n_src_layers} source layer(s)");
const BATCH: usize = 4096;
let n_workers = rayon::current_num_threads().min(16).max(4);
// At most 2 files open simultaneously: keeps n_workers-2 workers free
// for the Transform stage. Each open file monopolises one worker for the
// full duration of its read, so this must stay well below n_workers.
let max_open = 2;
let dst_filter = Arc::clone(&dst_layers);
let g = build_graph(
unitig_paths.into_iter(),
move |path: PathBuf, emit: &mut dyn FnMut(Vec<CanonicalKmer>)| -> OKIResult<()> {
let reader = UnitigFileReader::open_sequential(&path)?;
let mut batch: Vec<CanonicalKmer> = Vec::with_capacity(BATCH);
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
batch.push(kmer);
if batch.len() == BATCH {
emit(std::mem::replace(&mut batch, Vec::with_capacity(BATCH)));
}
}
if !batch.is_empty() {
emit(batch);
}
Ok(())
},
move |kmer| dst_filter.iter().all(|l| l.find_slot(kmer).is_none()),
n_workers,
max_open,
)?;
let any_new = g.len() > 0;
debug!(
"partition {i}: de Bruijn graph done — {} new kmers",
g.len()
);
// Build new layer from de Bruijn graph if there are new kmers.
let new_layer_idx = n_dst_layers;
let new_layer_dir = KmerLayer::new(&dst_partition, new_layer_idx)
.dir()
.to_path_buf();
let n_new = if any_new {
debug!("partition {i}: unitig traversal start — {} nodes", g.len());
let n_nodes = materialize_layer(g, &new_layer_dir, block_bits, evidence)?;
debug!("partition {i}: MPHF build done");
n_nodes
} else {
drop(g);
0
};
let t_open = std::time::Instant::now();
let new_mphf: Option<Arc<obikindex::layer::MphfOnly>> = if any_new {
Some(Arc::new(obikindex::layer::MphfOnly::open(&new_layer_dir)?))
} else {
None
};
debug!(
"partition {i}: MPHF open in {:.3}s",
t_open.elapsed().as_secs_f64()
);
let matrix_dir = |layer_dir: &Path| match mode {
MergeMode::Presence => layer_dir.join("presence"),
MergeMode::Count => layer_dir.join("counts"),
};
// ── New layer matrix: absent (dst-genome) columns only ────────────────
// Created and closed here; every source group below resumes it and
// appends its own columns, exactly like the existing dst layers.
let new_data_dir: Option<PathBuf> = if any_new {
let data_dir = matrix_dir(&new_layer_dir);
fs::create_dir_all(&data_dir)?;
let mut mb = MatrixBuilder::new(presence(mode), n_new, &data_dir).map_err(OKIError::Io)?;
for _ in 0..n_dst_genomes {
mb.add_absent_col().map_err(OKIError::Io)?;
}
mb.close().map_err(OKIError::Io)?;
Some(data_dir)
} else {
None
};
let exist_data_dirs: Vec<PathBuf> = dst_layers.iter().map(|l| matrix_dir(l.dir())).collect();
// ── Pass 2, by consecutive groups of sources ──────────────────────────
//
// Only one group's source layers and destination columns are alive at a
// time, so the number of live mappings per partition no longer grows
// with the number of sources. SRC_GROUP matches `max_open`: pass 2
// never streams more than `max_open` unitigs.bin files at once, so a
// larger group would only keep more layers/columns open without adding
// read parallelism.
//
// Invariant: groups are visited in source order and each appends its
// `group_n` columns in source order, so the absolute column of genome
// `g` of a source is `n_dst_genomes + col_base + offset_local + g` —
// the same column as when every source was processed in one session.
//
// Memory: the group's columns are built in RAM (`add_mem_col`), at most
// `SRC_GROUP × src_n × (L_dst + 1)` live columns per partition, each of
// `n_layer / 8` bytes (presence) or `n_layer` bytes + overflow (counts):
// about `SRC_GROUP × src_n × N_part × (1/8 | 1)` bytes, `N_part` being
// the partition's k-mer count over all dst layers. For mono-genome
// sources this stays below pass 1's de Bruijn graph (≥ ~19 B per new
// k-mer) unless the new layer is tiny next to the existing ones
// (incremental merge into a large index).
const SRC_GROUP: usize = 2;
let t_pass2 = std::time::Instant::now();
let mut col_base = 0usize;
for group in sources.chunks(SRC_GROUP) {
let group_n: usize = group.iter().map(|(_, n)| *n).sum();
// This group's source layers, for this partition only. Opened
// before any builder so errors propagate via `?` before any worker
// thread is spawned. `col_offset` is local to the group.
let mut items: Vec<(usize, usize, PathBuf, Arc<KmerLayer>)> = Vec::new();
let mut col_offset = 0usize;
for (src, src_n) in group {
let partition = src.partition(i)?;
for l in 0..partition.n_layers() {
let layer = Arc::new(partition.layer(l)?.open()?);
let unitigs_path = layer.unitigs_path();
if unitigs_path.exists() {
items.push((col_offset, *src_n, unitigs_path, layer));
}
}
col_offset += src_n;
}
let expected_cols = n_dst_genomes + col_base;
let open_group = |data_dir: &Path| -> OKIResult<(MatrixBuilder, Vec<ColBuilder>)> {
let mut mb = MatrixBuilder::resume(presence(mode), data_dir).map_err(OKIError::Io)?;
if mb.n_cols() != expected_cols {
return Err(OKIError::InvalidData {
context: "merge pass2",
detail: format!(
"{}: {} columns, expected {expected_cols}",
data_dir.display(),
mb.n_cols()
),
});
}
// In-memory columns: no sparse file / `MmapMut` on the write
// path, so a block the FS cannot allocate is an `io::Error` at
// `close`, not a SIGBUS (see `MatrixBuilder::add_mem_col`).
let cols = (0..group_n).map(|_| mb.add_mem_col()).collect();
Ok((mb, cols))
};
let (exist_mbs, exist_cols): (Vec<MatrixBuilder>, Vec<Vec<ColBuilder>>) = exist_data_dirs
.iter()
.map(|d| open_group(d))
.collect::<OKIResult<Vec<_>>>()?
.into_iter()
.unzip();
let (new_mb, new_cols) = match &new_data_dir {
Some(d) => {
let (mb, cols) = open_group(d)?;
(Some(mb), cols)
}
None => (None, vec![]),
};
// `items` (and with it this group's source layers) is consumed by
// the pipeline and dropped when it returns.
let (exist_cols, new_cols) = run_pass2(
items,
&dst_layers,
&new_mphf,
exist_cols,
new_cols,
n_workers,
max_open,
)?;
for (mb, cols) in exist_mbs.into_iter().zip(exist_cols) {
for c in cols {
c.close()?;
}
mb.close().map_err(OKIError::Io)?;
}
for c in new_cols {
c.close()?;
}
if let Some(mb) = new_mb {
mb.close().map_err(OKIError::Io)?;
}
col_base += group_n;
}
debug!(
"partition {i}: pass2 done in {:.3}s",
t_pass2.elapsed().as_secs_f64()
);
Ok(n_new)
}