feat: add sparse on-disk format for presence matrices
The index packing API now accepts a `sparse` parameter to generate `PersistentSparseBitMatrix` files alongside existing dense matrices. The sibling cache automatically detects this format via an `is_multi.prsb` marker file and routes queries identically to the dense variant. A new `--sparse` CLI flag exposes the option, with tests verifying end-to-end pipeline correctness and storage equivalence.
This commit is contained in:
1 parent
50f4820cb9
commit
3ba26b3dc1
14 files changed
+279
-24
No files matched your search
@@ -22,7 +22,7 @@ mod sparse;
|
||||
pub use builder::PersistentBitMatrixBuilder;
|
||||
pub use packed::pack_bit_matrix;
|
||||
pub use persistent::PersistentBitMatrix;
|
||||
pub use sparse::{PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
|
||||
pub use sparse::{PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_sparse_bit_matrix};
|
||||
|
||||
pub(crate) use pairwise::{pairwise_matrix, pairwise2_matrix};
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
//! used by any production code path yet — a new type, not a replacement.
|
||||
//!
|
||||
//! On-disk layout (a directory, mirroring [`super::PersistentBitMatrix`]'s
|
||||
//! own directory-of-files convention): `meta.json` plus four components —
|
||||
//! own directory-of-files convention): `sparse_meta.json` plus four components —
|
||||
//! `is_multi.prsb` (rank-capable flag: singleton row vs. multi-genome
|
||||
//! row), `singleton.pfiv` (genome index, one entry per singleton row,
|
||||
//! `ceil(log2(n_cols))` bits each), `multi.pfiv` (`dict_id`, one entry per
|
||||
@@ -33,13 +33,14 @@ use crate::meta::field;
|
||||
use crate::rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
|
||||
|
||||
use super::PersistentBitMatrix;
|
||||
use super::packed::PackedBitMatrix;
|
||||
|
||||
fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") }
|
||||
fn singleton_path(dir: &Path) -> PathBuf { dir.join("singleton.pfiv") }
|
||||
fn multi_path(dir: &Path) -> PathBuf { dir.join("multi.pfiv") }
|
||||
fn dict_offsets_base(dir: &Path) -> PathBuf { dir.join("dict_offsets") }
|
||||
fn dict_values_path(dir: &Path) -> PathBuf { dir.join("dict_values.bin") }
|
||||
fn meta_path(dir: &Path) -> PathBuf { dir.join("meta.json") }
|
||||
fn meta_path(dir: &Path) -> PathBuf { dir.join("sparse_meta.json") }
|
||||
|
||||
struct SparseMeta {
|
||||
n: usize,
|
||||
@@ -53,7 +54,7 @@ impl SparseMeta {
|
||||
fn load(dir: &Path) -> io::Result<Self> {
|
||||
let s = fs::read_to_string(meta_path(dir))?;
|
||||
let get = |name: &str| {
|
||||
field(&s, name).ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, format!("bad meta.json: missing {name}")))
|
||||
field(&s, name).ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, format!("bad sparse_meta.json: missing {name}")))
|
||||
};
|
||||
Ok(Self {
|
||||
n: get("n")?,
|
||||
@@ -370,3 +371,30 @@ impl PersistentSparseBitMatrixBuilder {
|
||||
Ok(builder)
|
||||
}
|
||||
}
|
||||
|
||||
/// `pack --sparse`'s entry point: converts a presence directory into the
|
||||
/// sparse on-disk format in place, mirroring [`super::pack_bit_matrix`]'s
|
||||
/// convention (old-format files removed only after the new format is
|
||||
/// fully written, so a crash mid-conversion leaves the previous, still
|
||||
/// valid format rather than a half-written one). Idempotent — does
|
||||
/// nothing if `is_multi.prsb` already exists. Packs to dense
|
||||
/// (`matrix.pbmx`) first via [`super::pack_bit_matrix`] if that hasn't
|
||||
/// happened yet, since the dense→sparse transpose (`build_from_dense`)
|
||||
/// needs random row access, which only the packed/columnar dense forms
|
||||
/// give — not a further reason to keep the dense file around afterward.
|
||||
pub fn pack_sparse_bit_matrix(dir: &Path) -> io::Result<()> {
|
||||
if is_multi_path(dir).exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let packed_path = dir.join("matrix.pbmx");
|
||||
if !packed_path.exists() {
|
||||
super::pack_bit_matrix(dir)?;
|
||||
}
|
||||
|
||||
let dense = PersistentBitMatrix::Packed(PackedBitMatrix::open(&packed_path)?);
|
||||
PersistentSparseBitMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
||||
drop(dense);
|
||||
fs::remove_file(&packed_path)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -19,7 +19,7 @@ pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
|
||||
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
|
||||
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
|
||||
pub use eliasfano::{EliasFano, EliasFanoBuilder};
|
||||
pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_bit_matrix};
|
||||
pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix};
|
||||
pub use builder::PersistentCompactIntVecBuilder;
|
||||
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
|
||||
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
|
||||
|
||||
@@ -26,6 +26,14 @@ pub trait BinaryMatrix {
|
||||
fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<bool>]);
|
||||
/// Per-genome k-mer totals.
|
||||
fn count_ones(&self) -> Array1<u64>;
|
||||
|
||||
/// Allocating variant of [`fill_sub_matrix`](Self::fill_sub_matrix) —
|
||||
/// provided once here so implementers only need the fill-in-place form.
|
||||
fn sub_matrix(&self, slots: &[usize]) -> Vec<Vec<bool>> {
|
||||
let mut out: Vec<Vec<bool>> = (0..self.n_cols()).map(|_| Vec::new()).collect();
|
||||
self.fill_sub_matrix(slots, &mut out);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a Jaccard distance matrix (`1 - J`) into a Mash distance matrix, per
|
||||
|
||||
Reference in new issue
Block a user