diff --git a/src/obicompactvec/src/bitmatrix/mod.rs b/src/obicompactvec/src/bitmatrix/mod.rs index 1a69806d..610b88f1 100644 --- a/src/obicompactvec/src/bitmatrix/mod.rs +++ b/src/obicompactvec/src/bitmatrix/mod.rs @@ -18,6 +18,7 @@ mod packed; mod pairwise; mod persistent; mod sparse; +mod sparse_columnar; pub use builder::PersistentBitMatrixBuilder; pub use group_ops::batch_presence_counts; diff --git a/src/obicompactvec/src/bitmatrix/packed.rs b/src/obicompactvec/src/bitmatrix/packed.rs index 5451597d..ca51dfec 100644 --- a/src/obicompactvec/src/bitmatrix/packed.rs +++ b/src/obicompactvec/src/bitmatrix/packed.rs @@ -105,7 +105,7 @@ impl PackedBitMatrix { /// Reads just the `n_cols` field from an existing packed matrix's header, /// without mapping the file. Used by `pack_bit_matrix` to tell a genuinely /// complete pack from a stale one that predates a later column-widening. -fn packed_bit_matrix_n_cols(path: &Path) -> io::Result { +pub(super) fn packed_bit_matrix_n_cols(path: &Path) -> io::Result { let mut f = File::open(path)?; let mut header = [0u8; PBMX_HEADER]; f.read_exact(&mut header)?; diff --git a/src/obicompactvec/src/bitmatrix/sparse.rs b/src/obicompactvec/src/bitmatrix/sparse.rs index ba95a9b0..8d19d55a 100644 --- a/src/obicompactvec/src/bitmatrix/sparse.rs +++ b/src/obicompactvec/src/bitmatrix/sparse.rs @@ -31,11 +31,12 @@ use ndarray::{Array1, Array2}; use crate::traits::{BitPartials, ColumnWeights}; use crate::eliasfano::{EliasFano, EliasFanoBuilder}; use crate::fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range}; -use crate::meta::field; +use crate::meta::{MatrixMeta, field}; use crate::rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder}; use super::PersistentBitMatrix; -use super::packed::{PackedBitMatrix, pack_bit_matrix}; +use super::col_path; +use super::packed::{PackedBitMatrix, pack_bit_matrix, packed_bit_matrix_n_cols}; fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") } fn singleton_path(dir: &Path) -> PathBuf { dir.join("singleton.pfiv") } @@ -620,24 +621,38 @@ impl BitPartials for PersistentSparseBitMatrix { /// mid-conversion leaves a previous, still valid format. Idempotent: does /// nothing if `is_multi.prsb` already exists. /// -/// Always goes through the dense packed form, even right after a build. -/// That costs one extra write and read of the dense matrix, but keeps the -/// number of mappings at one instead of one per genome column. +/// Columnar data is transposed straight into the sparse format, a window of +/// rows at a time, one column file open at a time +/// ([`PersistentSparseBitMatrixBuilder::build_from_columnar`]): no dense +/// intermediate is written, and a worker holds no mapping of the columns +/// whatever their number. A dense `matrix.pbmx` is only read when it is the +/// current form of the layer: no columnar data pending, or a `matrix.pbmx` +/// that already has as many columns as the columnar metadata says. pub fn pack_sparse_bit_matrix(dir: &Path) -> io::Result<()> { if is_multi_path(dir).exists() { return Ok(()); } - - // Columnar → dense packed first, then dense packed → sparse. Opening the - // columnar matrix directly would map every column file at once (one - // mapping per genome, unbounded); `pack_bit_matrix` instead - // concatenates the column files one at a time, unmapped, and leaves a - // single `matrix.pbmx` to read from (one mapping). It is a no-op when - // the layer is already dense-packed, and re-packs a stale `matrix.pbmx` - // that predates newer columnar data. Every intermediate state is a - // valid format, so a crash leaves something this function resumes. - pack_bit_matrix(dir)?; let packed_path = dir.join("matrix.pbmx"); + + if let Ok(meta) = MatrixMeta::load(dir) { + if packed_bit_matrix_n_cols(&packed_path).ok() != Some(meta.n_cols) { + // Columnar data is the freshest form (any `matrix.pbmx` here is + // stale, e.g. copied from a merge base before the layer was + // widened): transpose it directly. + PersistentSparseBitMatrixBuilder::build_from_columnar(dir, &meta)?.close()?; + for c in 0..meta.n_cols { + fs::remove_file(col_path(dir, c))?; + } + fs::remove_file(dir.join("meta.json"))?; + let _ = fs::remove_file(&packed_path); + return Ok(()); + } + } + + // Dense packed form is the current one. `pack_bit_matrix` is a no-op when + // there is no columnar data left, and otherwise only drops the + // now-redundant column files. + pack_bit_matrix(dir)?; let dense = PersistentBitMatrix::Packed(PackedBitMatrix::open(&packed_path)?); PersistentSparseBitMatrixBuilder::build_from_dense(&dense, dir)?.close()?; drop(dense); diff --git a/src/obicompactvec/src/bitmatrix/sparse_columnar.rs b/src/obicompactvec/src/bitmatrix/sparse_columnar.rs new file mode 100644 index 00000000..08e45b7f --- /dev/null +++ b/src/obicompactvec/src/bitmatrix/sparse_columnar.rs @@ -0,0 +1,95 @@ +//! Building the sparse presence matrix straight from the columnar +//! `col_*.pbiv` files, with no dense intermediate and no mapping. +//! +//! Each column is opened, a window of it is read with `pread`, and it is +//! closed again: a worker holds at most one column file at a time, +//! whatever the number of genomes. See [`crate::transpose`]. + +use std::fs::File; +use std::io; +use std::os::unix::fs::FileExt; +use std::path::Path; + +use crate::bitvec::MAGIC; +use crate::meta::MatrixMeta; +use crate::transpose::{DEFAULT_MAX_HITS, DEFAULT_WINDOW_ROWS, Hit, transpose_windows}; +use crate::views::BitSliceView; + +use super::PersistentSparseBitMatrixBuilder; +use super::col_path; + +/// `PBIV` header: magic(4) + pad(4) + n(8). +const PBIV_HEADER: usize = 16; + +/// Reads the bits of `path` for rows `start..end` (`start` a multiple of 64) +/// into `words`, after checking the header against `n`. +fn read_bit_window( + path: &Path, + n: usize, + start: usize, + end: usize, + bytes: &mut Vec, + words: &mut Vec, +) -> io::Result<()> { + debug_assert_eq!(start % 64, 0); + let file = File::open(path)?; + let mut header = [0u8; PBIV_HEADER]; + file.read_exact_at(&mut header, 0)?; + if header[..4] != MAGIC || u64::from_le_bytes(header[8..16].try_into().unwrap()) as usize != n { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{}: not a PBIV column of {n} rows", path.display()), + )); + } + let nw = (end - start).div_ceil(64); + bytes.clear(); + bytes.resize(nw * 8, 0); + file.read_exact_at(bytes, (PBIV_HEADER + start / 8) as u64)?; + words.clear(); + words.extend(bytes.chunks_exact(8).map(|b| u64::from_le_bytes(b.try_into().unwrap()))); + Ok(()) +} + +impl PersistentSparseBitMatrixBuilder { + /// Builds the sparse matrix of the columnar layer in `dir` (`meta.json` + /// plus `col_*.pbiv`), writing the sparse files into `dir`. Same rows, + /// same genome order and so same bytes as going through a dense + /// `matrix.pbmx` (`build_from_dense`), without ever writing it. + pub fn build_from_columnar(dir: &Path, meta: &MatrixMeta) -> io::Result { + Self::build_from_columnar_with(dir, meta, DEFAULT_WINDOW_ROWS, DEFAULT_MAX_HITS) + } + + /// [`build_from_columnar`](Self::build_from_columnar) with explicit window + /// size (rows) and non-zero-cell cap; see [`crate::transpose`]. + pub fn build_from_columnar_with( + dir: &Path, + meta: &MatrixMeta, + window_rows: usize, + max_hits: usize, + ) -> io::Result { + let mut builder = Self::new(meta.n, meta.n_cols, dir)?; + let mut bytes: Vec = Vec::new(); + let mut words: Vec = Vec::new(); + transpose_windows( + meta.n, + meta.n_cols, + window_rows, + max_hits, + (), + |c, start, end, hits: &mut Vec>| { + read_bit_window(&col_path(dir, c), meta.n, start, end, &mut bytes, &mut words)?; + BitSliceView::new(&words, end - start).for_each_set_in(0..end - start, |r| { + hits.push((r as u32, c as u32, ())); + }); + Ok(()) + }, + |_, batch| { + for i in 0..batch.len() { + builder.push_row(batch.cols(i)); + } + Ok(()) + }, + )?; + Ok(builder) + } +} diff --git a/src/obicompactvec/src/intmatrix.rs b/src/obicompactvec/src/intmatrix.rs index 1b83ff1a..ea9f664b 100644 --- a/src/obicompactvec/src/intmatrix.rs +++ b/src/obicompactvec/src/intmatrix.rs @@ -298,7 +298,7 @@ impl PackedCompactIntMatrix { /// without mapping the file. Used by `pack_compact_int_matrix` to tell a /// genuinely complete pack from a stale one that predates a later /// column-widening. -fn packed_int_matrix_n_cols(path: &Path) -> io::Result { +pub(crate) fn packed_int_matrix_n_cols(path: &Path) -> io::Result { let mut f = File::open(path)?; let mut header = [0u8; PCMX_HEADER]; f.read_exact(&mut header)?; diff --git a/src/obicompactvec/src/lib.rs b/src/obicompactvec/src/lib.rs index 58a4ab75..89e01dd5 100644 --- a/src/obicompactvec/src/lib.rs +++ b/src/obicompactvec/src/lib.rs @@ -15,9 +15,11 @@ mod mmap_file; mod rankselect; mod reader; mod sparse_intmatrix; +mod sparse_intmatrix_columnar; mod storage_kind; mod tempbitvec; mod tempintvec; +mod transpose; pub mod traits; mod views; mod views_scan; diff --git a/src/obicompactvec/src/sparse_intmatrix.rs b/src/obicompactvec/src/sparse_intmatrix.rs index ed4b4eac..2061b4f0 100644 --- a/src/obicompactvec/src/sparse_intmatrix.rs +++ b/src/obicompactvec/src/sparse_intmatrix.rs @@ -33,6 +33,7 @@ use ndarray::{Array1, Array2}; use crate::bitmatrix::RowRank; use crate::builder::PersistentCompactIntVecBuilder; use crate::eliasfano::{EliasFano, EliasFanoBuilder}; +use crate::meta::MatrixMeta; use crate::reader::PersistentCompactIntVec; use crate::traits::{BitPartials, ColumnWeights, CountPartials}; use crate::{PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder}; @@ -618,22 +619,34 @@ impl PersistentSparseCompactIntMatrixBuilder { // ── pack_sparse_compact_int_matrix ────────────────────────────────────────── /// Converts a layer's count matrix (columnar `col_*.pciv` or dense -/// `matrix.pcmx`) to the sparse on-disk format in place, through the dense -/// packed form, exactly like [`crate::pack_sparse_bit_matrix`]. Old-format -/// files are removed only once the new format is fully written. -/// Idempotent: a no-op if the sparse format is already present. +/// `matrix.pcmx`) to the sparse on-disk format in place, exactly like +/// [`crate::pack_sparse_bit_matrix`]: columnar data is transposed straight +/// into the sparse format, window by window, one column file open at a time +/// (no dense intermediate, no mapping); a dense `matrix.pcmx` is read only +/// when it is the current form. Old-format files are removed only once the +/// new format is fully written. Idempotent: a no-op if the sparse format is +/// already present. pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> { - use crate::intmatrix::{PackedCompactIntMatrix, pack_compact_int_matrix}; + use crate::intmatrix::{PackedCompactIntMatrix, pack_compact_int_matrix, packed_int_matrix_n_cols}; if is_present(dir) { return Ok(()); } - - // Columnar → dense packed first, then dense packed → sparse: one - // mapping (`matrix.pcmx`) instead of one per column file. See - // `pack_sparse_bit_matrix` for the rationale and crash-safety argument. - pack_compact_int_matrix(dir)?; let packed_path = dir.join("matrix.pcmx"); + + if let Ok(meta) = MatrixMeta::load(dir) { + if packed_int_matrix_n_cols(&packed_path).ok() != Some(meta.n_cols) { + PersistentSparseCompactIntMatrixBuilder::build_from_columnar(dir, &meta)?.close()?; + for c in 0..meta.n_cols { + fs::remove_file(crate::intmatrix::col_path(dir, c))?; + } + fs::remove_file(dir.join("meta.json"))?; + let _ = fs::remove_file(&packed_path); + return Ok(()); + } + } + + pack_compact_int_matrix(dir)?; let dense = PersistentIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?); PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?; drop(dense); diff --git a/src/obicompactvec/src/sparse_intmatrix_columnar.rs b/src/obicompactvec/src/sparse_intmatrix_columnar.rs new file mode 100644 index 00000000..5abe9df5 --- /dev/null +++ b/src/obicompactvec/src/sparse_intmatrix_columnar.rs @@ -0,0 +1,148 @@ +//! Building the sparse count matrix straight from the columnar +//! `col_*.pciv` files, with no dense intermediate and no mapping. Counterpart +//! of `bitmatrix::sparse_columnar`; see [`crate::transpose`]. +//! +//! A `PCIV` column is `header(40) | primary(n bytes) | overflow(n_overflow +//! × 12 B, sorted by slot) | index`. A primary byte `255` is a sentinel: the +//! true value (≥ 255) is the overflow entry of that slot. A window therefore +//! reads its primary bytes, then looks up the overflow entries of its +//! sentinels (binary search for the first one, one contiguous read for the +//! rest). + +use std::fs::File; +use std::io; +use std::os::unix::fs::FileExt; +use std::path::Path; + +use crate::format::{HEADER_SIZE, MAGIC, OVERFLOW_ENTRY_SIZE, parse_overflow_entry}; +use crate::intmatrix::col_path; +use crate::meta::MatrixMeta; +use crate::sparse_intmatrix::PersistentSparseCompactIntMatrixBuilder; +use crate::transpose::{DEFAULT_MAX_HITS, DEFAULT_WINDOW_ROWS, Hit, transpose_windows}; + +fn invalid(path: &Path, what: &str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, format!("{}: {what}", path.display())) +} + +/// Appends the non-zero cells of column `c` (file `path`) for rows +/// `start..end` to `hits`, as `(row - start, c, value)`. +fn read_int_window( + path: &Path, + n: usize, + c: usize, + start: usize, + end: usize, + primary: &mut Vec, + hits: &mut Vec>, +) -> io::Result<()> { + let file = File::open(path)?; + let mut header = [0u8; HEADER_SIZE]; + file.read_exact_at(&mut header, 0)?; + if header[..4] != MAGIC || u64::from_le_bytes(header[8..16].try_into().unwrap()) as usize != n { + return Err(invalid(path, &format!("not a PCIV column of {n} rows"))); + } + let n_overflow = u64::from_le_bytes(header[16..24].try_into().unwrap()) as usize; + + primary.clear(); + primary.resize(end - start, 0); + file.read_exact_at(primary, (HEADER_SIZE + start) as u64)?; + + // Slots (relative to `start`) whose primary byte is the 255 sentinel, + // pushed to `hits` with a placeholder and patched below. + let first_hit = hits.len(); + let mut n_sentinels = 0usize; + let mut chunks = primary.chunks_exact(8); + let mut base = 0usize; + for chunk in &mut chunks { + if u64::from_le_bytes(chunk.try_into().unwrap()) != 0 { + for (j, &b) in chunk.iter().enumerate() { + if b != 0 { + n_sentinels += (b == 255) as usize; + hits.push(((base + j) as u32, c as u32, b as u32)); + } + } + } + base += 8; + } + for (j, &b) in chunks.remainder().iter().enumerate() { + if b != 0 { + n_sentinels += (b == 255) as usize; + hits.push(((base + j) as u32, c as u32, b as u32)); + } + } + if n_sentinels == 0 { + return Ok(()); + } + + // Overflow entries of this window: first entry with slot >= start, found + // by binary search over the on-disk section, then one contiguous read. + let ov_base = (HEADER_SIZE + n) as u64; + let slot_at = |i: usize| -> io::Result { + let mut b = [0u8; 8]; + file.read_exact_at(&mut b, ov_base + (i * OVERFLOW_ENTRY_SIZE) as u64)?; + Ok(u64::from_le_bytes(b) as usize) + }; + let (mut lo, mut hi) = (0usize, n_overflow); + while lo < hi { + let mid = lo + (hi - lo) / 2; + if slot_at(mid)? < start { + lo = mid + 1; + } else { + hi = mid; + } + } + if lo + n_sentinels > n_overflow { + return Err(invalid(path, "fewer overflow entries than sentinel bytes")); + } + let mut raw = vec![0u8; n_sentinels * OVERFLOW_ENTRY_SIZE]; + file.read_exact_at(&mut raw, ov_base + (lo * OVERFLOW_ENTRY_SIZE) as u64)?; + let mut k = 0usize; + for h in &mut hits[first_hit..] { + if h.2 == 255 { + let (slot, value) = parse_overflow_entry(&raw, 0, k); + if slot != start + h.0 as usize { + return Err(invalid(path, "overflow entry does not match its sentinel slot")); + } + h.2 = value; + k += 1; + } + } + Ok(()) +} + +impl PersistentSparseCompactIntMatrixBuilder { + /// Builds the sparse count matrix of the columnar layer in `dir` + /// (`meta.json` plus `col_*.pciv`), writing the sparse files into `dir`. + /// Same rows, same cells and so same bytes as going through a dense + /// `matrix.pcmx` (`build_from_dense`), without ever writing it. + pub fn build_from_columnar(dir: &Path, meta: &MatrixMeta) -> io::Result { + Self::build_from_columnar_with(dir, meta, DEFAULT_WINDOW_ROWS, DEFAULT_MAX_HITS) + } + + /// [`build_from_columnar`](Self::build_from_columnar) with explicit window + /// size (rows) and non-zero-cell cap; see [`crate::transpose`]. + pub fn build_from_columnar_with( + dir: &Path, + meta: &MatrixMeta, + window_rows: usize, + max_hits: usize, + ) -> io::Result { + let mut builder = Self::new(meta.n, meta.n_cols, dir)?; + let mut primary: Vec = Vec::new(); + transpose_windows( + meta.n, + meta.n_cols, + window_rows, + max_hits, + 0u32, + |c, start, end, hits| read_int_window(&col_path(dir, c), meta.n, c, start, end, &mut primary, hits), + |_, batch| { + for i in 0..batch.len() { + builder.push_row(batch.cols(i), batch.vals(i)); + } + Ok(()) + }, + )?; + Ok(builder) + } +} diff --git a/src/obicompactvec/src/tests/mod.rs b/src/obicompactvec/src/tests/mod.rs index 810ac805..c0d7361d 100644 --- a/src/obicompactvec/src/tests/mod.rs +++ b/src/obicompactvec/src/tests/mod.rs @@ -10,6 +10,7 @@ mod mem_builder; mod rankselect; mod sparse; mod sparse_intmatrix; +mod transpose_columnar; mod views_scan; use tempfile::tempdir; diff --git a/src/obicompactvec/src/tests/transpose_columnar.rs b/src/obicompactvec/src/tests/transpose_columnar.rs new file mode 100644 index 00000000..78cfdb14 --- /dev/null +++ b/src/obicompactvec/src/tests/transpose_columnar.rs @@ -0,0 +1,156 @@ +//! The windowed columnar → sparse transposition must produce the very bytes +//! the dense route (`pack` to `matrix.pbmx`/`.pcmx`, then `build_from_dense`) +//! produces, whatever the window size and non-zero-cell cap — including the +//! retry-with-a-smaller-window path. + +use std::fs; +use std::path::Path; + +use tempfile::tempdir; + +use crate::meta::MatrixMeta; +use crate::{ + PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder, + PersistentIntMatrix, PersistentSparseBitMatrixBuilder, PersistentSparseCompactIntMatrixBuilder, + pack_bit_matrix, pack_compact_int_matrix, +}; + +const N: usize = 5_000 + 37; // not a multiple of 64 +const N_COLS: usize = 7; + +fn xorshift(s: &mut u64) -> u64 { + *s ^= *s << 13; + *s ^= *s >> 7; + *s ^= *s << 17; + *s +} + +/// Column `c`: all zero, very sparse, sparse, dense, ... so that rows have +/// from 0 to `N_COLS` cells and whole words are zero in some columns. +fn cell(c: usize, s: &mut u64) -> Option { + let r = xorshift(s); + let keep = match c % 4 { + 0 => false, + 1 => r % 97 == 0, + 2 => r % 7 == 0, + _ => r % 2 == 0, + }; + keep.then(|| match (r >> 20) % 11 { + 0 => 255, + 1 => 256, + 2 => 70_000, + v => (v as u32) + 1, + }) +} + +fn build_bits(root: &Path) { + let mut b = PersistentBitMatrixBuilder::new(N, &root.join("presence")).unwrap(); + let mut s = 0x9E37_79B9_7F4A_7C15u64; + for c in 0..N_COLS { + let mut cb = b.add_col().unwrap(); + for slot in 0..N { + cb.set(slot, cell(c, &mut s).is_some()); + } + cb.close().unwrap(); + } + b.close().unwrap(); +} + +fn build_ints(root: &Path) { + let mut b = PersistentCompactIntMatrixBuilder::new(N, &root.join("counts")).unwrap(); + let mut s = 0x9E37_79B9_7F4A_7C15u64; + for c in 0..N_COLS { + let mut cb = b.add_col().unwrap(); + for slot in 0..N { + if let Some(v) = cell(c, &mut s) { + cb.set(slot, v); + } + } + cb.close().unwrap(); + } + b.close().unwrap(); +} + +/// Sparse-format files of `dir` (columnar leftovers and dense files excluded), +/// as `(name, bytes)` sorted by name. +fn sparse_files(dir: &Path) -> Vec<(String, Vec)> { + let mut v: Vec<(String, Vec)> = fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().path()) + .filter(|p| { + let n = p.file_name().unwrap().to_string_lossy().into_owned(); + !(n.starts_with("col_") || n == "meta.json" || n.starts_with("matrix.")) + }) + .map(|p| (p.file_name().unwrap().to_string_lossy().into_owned(), fs::read(&p).unwrap())) + .collect(); + v.sort(); + v +} + +const WINDOWS: &[(usize, usize)] = &[ + (64, usize::MAX / 4), // smallest window, never too dense + (192, 50), // forces the retry-with-half-window path + (4096, 10_000), // a few windows + (crate::transpose::DEFAULT_WINDOW_ROWS, crate::transpose::DEFAULT_MAX_HITS), +]; + +#[test] +fn bit_columnar_transpose_is_byte_identical_to_the_dense_route() { + let ref_root = tempdir().unwrap(); + build_bits(ref_root.path()); + let ref_dir = ref_root.path().join("presence"); + pack_bit_matrix(&ref_dir).unwrap(); + let dense = PersistentBitMatrix::open(ref_root.path()).unwrap(); + PersistentSparseBitMatrixBuilder::build_from_dense(&dense, &ref_dir).unwrap().close().unwrap(); + drop(dense); + let expected = sparse_files(&ref_dir); + assert!(!expected.is_empty()); + + for &(window, max_hits) in WINDOWS { + let root = tempdir().unwrap(); + build_bits(root.path()); + let dir = root.path().join("presence"); + let meta = MatrixMeta::load(&dir).unwrap(); + PersistentSparseBitMatrixBuilder::build_from_columnar_with(&dir, &meta, window, max_hits) + .unwrap() + .close() + .unwrap(); + assert_eq!(sparse_files(&dir), expected, "window={window} max_hits={max_hits}"); + } +} + +#[test] +fn int_columnar_transpose_is_byte_identical_to_the_dense_route() { + let ref_root = tempdir().unwrap(); + build_ints(ref_root.path()); + let ref_dir = ref_root.path().join("counts"); + pack_compact_int_matrix(&ref_dir).unwrap(); + let dense = PersistentIntMatrix::open(ref_root.path()).unwrap(); + PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, &ref_dir).unwrap().close().unwrap(); + drop(dense); + let expected = sparse_files(&ref_dir); + assert!(!expected.is_empty()); + + for &(window, max_hits) in WINDOWS { + let root = tempdir().unwrap(); + build_ints(root.path()); + let dir = root.path().join("counts"); + let meta = MatrixMeta::load(&dir).unwrap(); + PersistentSparseCompactIntMatrixBuilder::build_from_columnar_with(&dir, &meta, window, max_hits) + .unwrap() + .close() + .unwrap(); + assert_eq!(sparse_files(&dir), expected, "window={window} max_hits={max_hits}"); + } +} + +#[test] +fn columnar_transpose_rejects_a_column_of_another_length() { + let root = tempdir().unwrap(); + build_bits(root.path()); + let dir = root.path().join("presence"); + let mut meta = MatrixMeta::load(&dir).unwrap(); + meta.n += 64; // metadata claims more rows than the column files hold + let err = PersistentSparseBitMatrixBuilder::build_from_columnar(&dir, &meta).err().expect("must fail"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); +} diff --git a/src/obicompactvec/src/transpose.rs b/src/obicompactvec/src/transpose.rs new file mode 100644 index 00000000..636ced8a --- /dev/null +++ b/src/obicompactvec/src/transpose.rs @@ -0,0 +1,136 @@ +//! Windowed column → row transposition, shared by the presence and count +//! packers. +//! +//! A columnar matrix stores one file per genome; the sparse formats are +//! built row by row, each row needing the cells of *every* column. Holding +//! every column mapped (or open) at once costs one mapping (or descriptor) +//! per column and per worker. This module never holds more than one column: +//! it cuts the rows into windows and, for each window, visits the columns +//! one after the other through a caller-supplied reader, collecting the +//! non-zero cells; the cells are then regrouped by row and handed to the +//! caller. No intermediate dense matrix is written. +//! +//! Memory is bounded by `max_hits`, not by the partition size: when a window +//! turns out denser than expected it is abandoned and retried with half as +//! many rows. + +use std::io; + +/// Default rows per window. A multiple of 64, so that a bit column's window +/// starts on a word boundary. 2 M rows = 256 KiB per bit column and read. +pub const DEFAULT_WINDOW_ROWS: usize = 1 << 21; + +/// Default cap on the non-zero cells held for one window. +pub const DEFAULT_MAX_HITS: usize = 4 << 20; + +/// Smallest window the retry logic shrinks to. +const MIN_WINDOW_ROWS: usize = 64; + +/// One non-zero cell of a window: row relative to the window start, column, +/// value. `V = ()` for presence matrices. +pub(crate) type Hit = (u32, u32, V); + +/// The rows of one window, regrouped by row, columns ascending within a row. +pub(crate) struct RowBatch { + offsets: Vec, + cols: Vec, + vals: Vec, +} + +impl RowBatch { + fn new() -> Self { + Self { offsets: Vec::new(), cols: Vec::new(), vals: Vec::new() } + } + + /// Number of rows in the window. + pub(crate) fn len(&self) -> usize { + self.offsets.len().saturating_sub(1) + } + + /// Columns of row `i`, ascending. + pub(crate) fn cols(&self, i: usize) -> &[u32] { + &self.cols[self.offsets[i] as usize..self.offsets[i + 1] as usize] + } + + /// Values of row `i`, in the order of [`cols`](Self::cols). + pub(crate) fn vals(&self, i: usize) -> &[V] { + &self.vals[self.offsets[i] as usize..self.offsets[i + 1] as usize] + } + + /// Stable counting sort of `hits` by row. `hits` was produced column by + /// column in ascending column order, so stability leaves each row's + /// columns ascending, as `push_row` requires, without a per-row sort. + fn fill(&mut self, rows: usize, hits: &[Hit], zero: V) { + self.offsets.clear(); + self.offsets.resize(rows + 1, 0); + for &(r, _, _) in hits { + self.offsets[r as usize + 1] += 1; + } + for i in 0..rows { + self.offsets[i + 1] += self.offsets[i]; + } + self.cols.clear(); + self.cols.resize(hits.len(), 0); + self.vals.clear(); + self.vals.resize(hits.len(), zero); + // `next[r]` = write position of the next cell of row `r`; reuse the + // tail of `offsets` would alias, so keep a small separate cursor. + let mut next: Vec = self.offsets[..rows].to_vec(); + for &(r, c, v) in hits { + let p = next[r as usize] as usize; + self.cols[p] = c; + self.vals[p] = v; + next[r as usize] += 1; + } + } +} + +/// Transposes an `n`-row, `n_cols`-column matrix window by window. +/// +/// `read_col(c, start, end, hits)` must append to `hits` every non-zero cell +/// of column `c` in rows `start..end`, as `(row - start, c, value)`. +/// `on_batch(start, batch)` receives each completed window, in ascending row +/// order; `batch` row `i` is matrix row `start + i`. +/// +/// `window_rows` is rounded down to a multiple of 64 (minimum 64), and `start` +/// is always a multiple of 64. +pub(crate) fn transpose_windows( + n: usize, + n_cols: usize, + window_rows: usize, + max_hits: usize, + zero: V, + mut read_col: impl FnMut(usize, usize, usize, &mut Vec>) -> io::Result<()>, + mut on_batch: impl FnMut(usize, &RowBatch) -> io::Result<()>, +) -> io::Result<()> { + let window_max = (window_rows / 64 * 64).clamp(MIN_WINDOW_ROWS, u32::MAX as usize / 2); + // `Hit` rows are u32 and offsets count hits in u32. + let max_hits = max_hits.min(u32::MAX as usize / 2); + let mut w = window_max; + let mut hits: Vec> = Vec::new(); + let mut batch = RowBatch::new(); + let mut start = 0usize; + while start < n { + let end = (start + w).min(n); + hits.clear(); + let mut too_dense = false; + for c in 0..n_cols { + read_col(c, start, end, &mut hits)?; + if hits.len() > max_hits && w > MIN_WINDOW_ROWS { + too_dense = true; + break; + } + } + if too_dense { + w = (w / 2 / 64 * 64).max(MIN_WINDOW_ROWS); + continue; + } + batch.fill(end - start, &hits, zero); + on_batch(start, &batch)?; + start = end; + if hits.len() < max_hits / 4 && w < window_max { + w = (w * 2).min(window_max); + } + } + Ok(()) +}