diff --git a/DevDocMD/implementation/merge_mmap_audit.md b/DevDocMD/implementation/merge_mmap_audit.md index 6625624b..c78a9d72 100644 --- a/DevDocMD/implementation/merge_mmap_audit.md +++ b/DevDocMD/implementation/merge_mmap_audit.md @@ -81,7 +81,7 @@ Both pass after every step. The existing `global_index_presence` (October binary B would take the 8 down to ~2. -Not covered here, and possibly the next limit: the sparse `pack` at finalize maps every column of a layer at once (see `merge_scaling.md`). +**Follow-up — sparse pack (same day).** `pack_sparse_bit_matrix`/`pack_sparse_compact_int_matrix` used to open the columnar matrix directly, which maps every column file of a layer at once: `n_genomes` maps per partition, since layers are packed one after another. Both now call the dense pack first. It concatenates the column files one at a time without mapping them, so the sparse matrix is built from a single `matrix.pbmx`/`.pcmx` mapping. This deliberately reverses an earlier choice ("never materialise a packed copy just to read it back once"). It costs one extra write and read of the dense matrix: `pack` went from 1.11 s to 1.60 s on the benchmark (single run). As a side effect, a stale `matrix.pbmx` that predates newer columnar data is now re-packed instead of being used as is (`pack_bit_matrix` compares `n_cols`). Both oracles pass. ## Known issue (found while building the oracle, not fixed) diff --git a/DevDocMD/implementation/merge_scaling.md b/DevDocMD/implementation/merge_scaling.md index cd723aa1..4b1b63d5 100644 --- a/DevDocMD/implementation/merge_scaling.md +++ b/DevDocMD/implementation/merge_scaling.md @@ -10,7 +10,7 @@ Open design question raised during the mmap refactor (`merge_mmap_audit.md`). Ta | Pass-2 column RAM | `SRC_GROUP × src_n × N_part × (1/8 \| 1)` B | bounded, see below | | Pass-1 de Bruijn graph | ≥ ~19 B per *new* k-mer (`HashMap`, verified in `obidebruinj/src/debruijn/graph.rs`) | unchanged | | Columnar intermediate files | `n_genomes × n_layers × n_partitions` (6 000 × 2 × 256 ≈ 3 M files, estimate) | **does not scale** | -| Sparse `pack` | maps **all** columns of a layer at once (`ColumnarBitMatrix::open`, verified) → ~`n_genomes × n_layers` maps per partition | **does not scale**; with 186 genomes × 2 layers × 192 concurrent partitions ≈ 71 k maps, already above `vm.max_map_count` (estimate, assumes 1 map per column and pack concurrency = cores) | +| Sparse `pack` | used to map **all** columns of a layer at once (`ColumnarBitMatrix::open`); layers are packed one after another, so `n_genomes` maps per partition (186 × 192 concurrent ≈ 36 k, estimate) | **fixed 2026-11-17**: now goes columnar → dense `matrix.pbmx` (sequential concatenation, unmapped) → sparse, one mapping per partition | `N_part`: k-mers of one dst partition over all layers. Pass-2 column RAM stays below pass-1 graph RAM for one-shot merges. It dominates only for **incremental** merges, where few genomes are added to a large index: the new layer is small, the existing layers are large. Example (estimate): `N_total = 1e12`, counts, 192 concurrent partitions → ~1.5 TB. A memory-budgeted concurrency cap is needed there whatever the column strategy. `obisys::MemoryBudget` exists but is no longer used by merge. diff --git a/scripts/bench_merge_oracle.sh b/scripts/bench_merge_oracle.sh index 5ac3a5bb..cb95867b 100755 --- a/scripts/bench_merge_oracle.sh +++ b/scripts/bench_merge_oracle.sh @@ -21,7 +21,7 @@ # difference there is reported, not judged) # --check DIR: merges into a scratch directory, compares the hash with # DIR/dump.sha256 (full diff against the zstd copy only on mismatch), -# and prints merge_partitions wall/RSS before → after +# and prints merge_partitions wall/RSS and pack wall before → after # # Usage: # scripts/bench_merge_oracle.sh (--record DIR | --check DIR) [--keep] @@ -81,9 +81,9 @@ sorted_dump() { # sorted_dump INDEX OUT_TXT — header first, rows sorted hash_of() { sha256sum "$1" | cut -d' ' -f1; } -stage_field() { # stage_field LOG FIELD — FIELD of the 'done stage=merge_partitions' line +stage_field() { # stage_field LOG FIELD [STAGE] — FIELD of the 'done stage=STAGE' line (default merge_partitions) perl -pe 's/\e\[[0-9;]*m//g' "$1" \ - | grep 'done stage=merge_partitions' \ + | grep "done stage=${3:-merge_partitions} " \ | sed -n "s/.*$2=\([^ ]*\( [KMG]*B\)\{0,1\}\).*/\1/p" | tail -n 1 } @@ -107,7 +107,8 @@ if ! "$BIN" merge --output "$MERGED" --force --force-presence "${SOURCES[@]}" \ fi WALL="$(stage_field "$LOG" wall_secs)" RSS="$(stage_field "$LOG" rss)" -echo "merge_partitions: wall=${WALL}s rss=${RSS}" +PACK_WALL="$(stage_field "$LOG" wall_secs pack)" +echo "merge_partitions: wall=${WALL}s rss=${RSS} pack: wall=${PACK_WALL}s" # ── 3. sorted dump + hash ─────────────────────────────────────────────────── DUMP="$WORK/dump.sorted.csv" @@ -140,7 +141,9 @@ fi REF_H="$(cat "$DIR/dump.sha256")" REF_WALL="$(stage_field "$DIR/merge.stderr.log" wall_secs)" REF_RSS="$(stage_field "$DIR/merge.stderr.log" rss)" +REF_PACK="$(stage_field "$DIR/merge.stderr.log" wall_secs pack)" echo "merge_partitions wall: ${REF_WALL}s → ${WALL}s rss: ${REF_RSS} → ${RSS}" +echo "pack wall: ${REF_PACK}s → ${PACK_WALL}s" python3 -c "import sys; r,n=float(sys.argv[1]),float(sys.argv[2]); print(f'merge_partitions wall ratio: {n/r:.2f}' + (' (> 1.20: SLOWER THAN 20 %)' if n > 1.2*r else ''))" \ "$REF_WALL" "$WALL" || true diff --git a/src/Cargo.lock b/src/Cargo.lock index 2687bce5..337b95e7 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1537,7 +1537,7 @@ dependencies = [ [[package]] name = "obikmer" -version = "1.4.3" +version = "1.4.4" dependencies = [ "clap", "csv", diff --git a/src/obicompactvec/src/bitmatrix/sparse.rs b/src/obicompactvec/src/bitmatrix/sparse.rs index 9513e784..89eeb496 100644 --- a/src/obicompactvec/src/bitmatrix/sparse.rs +++ b/src/obicompactvec/src/bitmatrix/sparse.rs @@ -35,9 +35,7 @@ use crate::meta::field; use crate::rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder}; use super::PersistentBitMatrix; -use super::col_path; -use super::columnar::ColumnarBitMatrix; -use super::packed::PackedBitMatrix; +use super::packed::{PackedBitMatrix, pack_bit_matrix}; use crate::views::nonzero_triples; fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") } @@ -615,37 +613,33 @@ impl BitPartials for PersistentSparseBitMatrix { m } } -/// sparse on-disk format in place, mirroring [`super::pack_bit_matrix`]'s -/// convention (old-format files removed only after the new format is -/// fully written, so a crash mid-conversion leaves the previous, still -/// valid format rather than a half-written one). Idempotent — does + +/// Converts a layer's presence matrix (columnar `col_*.pbiv` or dense +/// `matrix.pbmx`) to the sparse on-disk format in place. Old-format files +/// are removed only after the new format is fully written, so a crash +/// mid-conversion leaves a previous, still valid format. Idempotent: does /// nothing if `is_multi.prsb` already exists. /// -/// The dense→sparse transpose (`build_from_dense`) only needs `fill_row`, -/// which costs the same `O(n_cols)` per row on `Columnar` as on `Packed` -/// (see `ColumnarBitMatrix::fill_row`) — so this reads straight from -/// whichever dense form is already on disk. In particular, when this runs -/// right after a build (the common case: `presence/` is still `Columnar`, -/// `matrix.pbmx` doesn't exist yet), it never materialises a full packed -/// copy of the presence matrix just to read it back once and delete it. +/// Always goes through the dense packed form, even right after a build. +/// That costs one extra write and read of the dense matrix, but keeps the +/// number of mappings at one instead of one per genome column. pub fn pack_sparse_bit_matrix(dir: &Path) -> io::Result<()> { if is_multi_path(dir).exists() { return Ok(()); } + // Columnar → dense packed first, then dense packed → sparse. Opening the + // columnar matrix directly would map every column file at once (one + // mapping per genome, unbounded); `pack_bit_matrix` instead + // concatenates the column files one at a time, unmapped, and leaves a + // single `matrix.pbmx` to read from (one mapping). It is a no-op when + // the layer is already dense-packed, and re-packs a stale `matrix.pbmx` + // that predates newer columnar data. Every intermediate state is a + // valid format, so a crash leaves something this function resumes. + pack_bit_matrix(dir)?; let packed_path = dir.join("matrix.pbmx"); - if packed_path.exists() { - let dense = PersistentBitMatrix::Packed(PackedBitMatrix::open(&packed_path)?); - PersistentSparseBitMatrixBuilder::build_from_dense(&dense, dir)?.close()?; - drop(dense); - fs::remove_file(&packed_path)?; - } else { - let dense = PersistentBitMatrix::Columnar(ColumnarBitMatrix::open(dir)?); - let n_cols = dense.n_cols(); - PersistentSparseBitMatrixBuilder::build_from_dense(&dense, dir)?.close()?; - drop(dense); - for c in 0..n_cols { let _ = fs::remove_file(col_path(dir, c)); } - let _ = fs::remove_file(dir.join("meta.json")); - } - Ok(()) + let dense = PersistentBitMatrix::Packed(PackedBitMatrix::open(&packed_path)?); + PersistentSparseBitMatrixBuilder::build_from_dense(&dense, dir)?.close()?; + drop(dense); + fs::remove_file(&packed_path) } diff --git a/src/obicompactvec/src/sparse_intmatrix.rs b/src/obicompactvec/src/sparse_intmatrix.rs index e2534cc7..ed4b4eac 100644 --- a/src/obicompactvec/src/sparse_intmatrix.rs +++ b/src/obicompactvec/src/sparse_intmatrix.rs @@ -617,37 +617,25 @@ impl PersistentSparseCompactIntMatrixBuilder { // ── pack_sparse_compact_int_matrix ────────────────────────────────────────── -/// Converts whichever dense count-matrix format is on disk at `dir` -/// (`Packed`, i.e. `matrix.pcmx`, or `Columnar`, i.e. `col_*.pciv` + -/// `meta.json`) into the sparse on-disk format in place — generic over the -/// dense source the same way [`super::bitmatrix::pack_sparse_bit_matrix`] -/// is: detects the format, transposes straight from it, and only removes -/// the old dense files once the new format is fully written (a crash -/// mid-conversion leaves the previous, still-valid format rather than a -/// half-written one). Idempotent — a no-op if `singleton_values.pciv` -/// already exists. +/// Converts a layer's count matrix (columnar `col_*.pciv` or dense +/// `matrix.pcmx`) to the sparse on-disk format in place, through the dense +/// packed form, exactly like [`crate::pack_sparse_bit_matrix`]. Old-format +/// files are removed only once the new format is fully written. +/// Idempotent: a no-op if the sparse format is already present. pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> { - use crate::intmatrix::{ColumnarCompactIntMatrix, PackedCompactIntMatrix, col_path}; + use crate::intmatrix::{PackedCompactIntMatrix, pack_compact_int_matrix}; if is_present(dir) { return Ok(()); } + // Columnar → dense packed first, then dense packed → sparse: one + // mapping (`matrix.pcmx`) instead of one per column file. See + // `pack_sparse_bit_matrix` for the rationale and crash-safety argument. + pack_compact_int_matrix(dir)?; let packed_path = dir.join("matrix.pcmx"); - if packed_path.exists() { - let dense = PersistentIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?); - PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?; - drop(dense); - fs::remove_file(&packed_path)?; - } else { - let dense = PersistentIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?); - let n_cols = dense.n_cols(); - PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?; - drop(dense); - for c in 0..n_cols { - let _ = fs::remove_file(col_path(dir, c)); - } - let _ = fs::remove_file(dir.join("meta.json")); - } - Ok(()) + let dense = PersistentIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?); + PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?; + drop(dense); + fs::remove_file(&packed_path) } diff --git a/src/obikmer/Cargo.toml b/src/obikmer/Cargo.toml index a92a23b1..f212ecf4 100644 --- a/src/obikmer/Cargo.toml +++ b/src/obikmer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "obikmer" -version = "1.4.3" +version = "1.4.4" edition = "2024" [[bin]]