docs: update sparse packing strategy and benchmark scripts
Update documentation to reflect that sparse packing now maps all columns of a layer at once and packs layers sequentially, reversing the previous choice to avoid materializing a packed copy. This change introduces an extra write/read cycle for the dense matrix, increasing pack benchmark time from 1.11s to 1.60s. - Modify `pack_sparse_bit_matrix` in `src/obicompactvec/src/bitmatrix/sparse.rs` to always convert via the dense packed format, simplifying logic by removing conditional handling of existing files and ensuring a single mapping. - Update `pack_sparse_compact_int_matrix` in `src/obicompactvec/src/sparse_intmatrix.rs` to exclusively handle conversion from the dense packed format (`matrix.pcmx`) to sparse, removing columnar format handling and simplifying file cleanup. - Adjust `scripts/bench_merge_oracle.sh` to extract `PACK_WALL` metrics via `stage_field` for merge operations, updating grep patterns and output messages to report pack wall time. - Revise `DevDocMD/implementation/merge_mmap_audit.md` and `merge_scaling.md` to document the new sequential packing strategy, noting that `N_part` scaling applies only to incremental merges and that `obisys::MemoryBudget` is no longer used by merge. - Bump package version in `src/obikmer/Cargo.toml` from 1.4.3 to 1.4.4.
This commit is contained in:
1 parent
1f199c69db
commit
07bc5fc634
7 files changed
+47
-62
No files matched your search
Generated
+1
-1
@@ -1537,7 +1537,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "obikmer"
|
||||
version = "1.4.3"
|
||||
version = "1.4.4"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"csv",
|
||||
|
||||
@@ -35,9 +35,7 @@ use crate::meta::field;
|
||||
use crate::rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
|
||||
|
||||
use super::PersistentBitMatrix;
|
||||
use super::col_path;
|
||||
use super::columnar::ColumnarBitMatrix;
|
||||
use super::packed::PackedBitMatrix;
|
||||
use super::packed::{PackedBitMatrix, pack_bit_matrix};
|
||||
use crate::views::nonzero_triples;
|
||||
|
||||
fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") }
|
||||
@@ -615,37 +613,33 @@ impl BitPartials for PersistentSparseBitMatrix {
|
||||
m
|
||||
}
|
||||
}
|
||||
/// sparse on-disk format in place, mirroring [`super::pack_bit_matrix`]'s
|
||||
/// convention (old-format files removed only after the new format is
|
||||
/// fully written, so a crash mid-conversion leaves the previous, still
|
||||
/// valid format rather than a half-written one). Idempotent — does
|
||||
|
||||
/// Converts a layer's presence matrix (columnar `col_*.pbiv` or dense
|
||||
/// `matrix.pbmx`) to the sparse on-disk format in place. Old-format files
|
||||
/// are removed only after the new format is fully written, so a crash
|
||||
/// mid-conversion leaves a previous, still valid format. Idempotent: does
|
||||
/// nothing if `is_multi.prsb` already exists.
|
||||
///
|
||||
/// The dense→sparse transpose (`build_from_dense`) only needs `fill_row`,
|
||||
/// which costs the same `O(n_cols)` per row on `Columnar` as on `Packed`
|
||||
/// (see `ColumnarBitMatrix::fill_row`) — so this reads straight from
|
||||
/// whichever dense form is already on disk. In particular, when this runs
|
||||
/// right after a build (the common case: `presence/` is still `Columnar`,
|
||||
/// `matrix.pbmx` doesn't exist yet), it never materialises a full packed
|
||||
/// copy of the presence matrix just to read it back once and delete it.
|
||||
/// Always goes through the dense packed form, even right after a build.
|
||||
/// That costs one extra write and read of the dense matrix, but keeps the
|
||||
/// number of mappings at one instead of one per genome column.
|
||||
pub fn pack_sparse_bit_matrix(dir: &Path) -> io::Result<()> {
|
||||
if is_multi_path(dir).exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Columnar → dense packed first, then dense packed → sparse. Opening the
|
||||
// columnar matrix directly would map every column file at once (one
|
||||
// mapping per genome, unbounded); `pack_bit_matrix` instead
|
||||
// concatenates the column files one at a time, unmapped, and leaves a
|
||||
// single `matrix.pbmx` to read from (one mapping). It is a no-op when
|
||||
// the layer is already dense-packed, and re-packs a stale `matrix.pbmx`
|
||||
// that predates newer columnar data. Every intermediate state is a
|
||||
// valid format, so a crash leaves something this function resumes.
|
||||
pack_bit_matrix(dir)?;
|
||||
let packed_path = dir.join("matrix.pbmx");
|
||||
if packed_path.exists() {
|
||||
let dense = PersistentBitMatrix::Packed(PackedBitMatrix::open(&packed_path)?);
|
||||
PersistentSparseBitMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
||||
drop(dense);
|
||||
fs::remove_file(&packed_path)?;
|
||||
} else {
|
||||
let dense = PersistentBitMatrix::Columnar(ColumnarBitMatrix::open(dir)?);
|
||||
let n_cols = dense.n_cols();
|
||||
PersistentSparseBitMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
||||
drop(dense);
|
||||
for c in 0..n_cols { let _ = fs::remove_file(col_path(dir, c)); }
|
||||
let _ = fs::remove_file(dir.join("meta.json"));
|
||||
}
|
||||
Ok(())
|
||||
let dense = PersistentBitMatrix::Packed(PackedBitMatrix::open(&packed_path)?);
|
||||
PersistentSparseBitMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
||||
drop(dense);
|
||||
fs::remove_file(&packed_path)
|
||||
}
|
||||
@@ -617,37 +617,25 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
||||
|
||||
// ── pack_sparse_compact_int_matrix ──────────────────────────────────────────
|
||||
|
||||
/// Converts whichever dense count-matrix format is on disk at `dir`
|
||||
/// (`Packed`, i.e. `matrix.pcmx`, or `Columnar`, i.e. `col_*.pciv` +
|
||||
/// `meta.json`) into the sparse on-disk format in place — generic over the
|
||||
/// dense source the same way [`super::bitmatrix::pack_sparse_bit_matrix`]
|
||||
/// is: detects the format, transposes straight from it, and only removes
|
||||
/// the old dense files once the new format is fully written (a crash
|
||||
/// mid-conversion leaves the previous, still-valid format rather than a
|
||||
/// half-written one). Idempotent — a no-op if `singleton_values.pciv`
|
||||
/// already exists.
|
||||
/// Converts a layer's count matrix (columnar `col_*.pciv` or dense
|
||||
/// `matrix.pcmx`) to the sparse on-disk format in place, through the dense
|
||||
/// packed form, exactly like [`crate::pack_sparse_bit_matrix`]. Old-format
|
||||
/// files are removed only once the new format is fully written.
|
||||
/// Idempotent: a no-op if the sparse format is already present.
|
||||
pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
||||
use crate::intmatrix::{ColumnarCompactIntMatrix, PackedCompactIntMatrix, col_path};
|
||||
use crate::intmatrix::{PackedCompactIntMatrix, pack_compact_int_matrix};
|
||||
|
||||
if is_present(dir) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Columnar → dense packed first, then dense packed → sparse: one
|
||||
// mapping (`matrix.pcmx`) instead of one per column file. See
|
||||
// `pack_sparse_bit_matrix` for the rationale and crash-safety argument.
|
||||
pack_compact_int_matrix(dir)?;
|
||||
let packed_path = dir.join("matrix.pcmx");
|
||||
if packed_path.exists() {
|
||||
let dense = PersistentIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?);
|
||||
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
||||
drop(dense);
|
||||
fs::remove_file(&packed_path)?;
|
||||
} else {
|
||||
let dense = PersistentIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?);
|
||||
let n_cols = dense.n_cols();
|
||||
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
||||
drop(dense);
|
||||
for c in 0..n_cols {
|
||||
let _ = fs::remove_file(col_path(dir, c));
|
||||
}
|
||||
let _ = fs::remove_file(dir.join("meta.json"));
|
||||
}
|
||||
Ok(())
|
||||
let dense = PersistentIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?);
|
||||
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
||||
drop(dense);
|
||||
fs::remove_file(&packed_path)
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "obikmer"
|
||||
version = "1.4.3"
|
||||
version = "1.4.4"
|
||||
edition = "2024"
|
||||
|
||||
[[bin]]
|
||||
|
||||
Reference in new issue
Block a user