feat: add sparse on-disk format for presence matrices

The index packing API now accepts a `sparse` parameter to generate `PersistentSparseBitMatrix` files alongside existing dense matrices. The sibling cache automatically detects this format via an `is_multi.prsb` marker file and routes queries identically to the dense variant. A new `--sparse` CLI flag exposes the option, with tests verifying end-to-end pipeline correctness and storage equivalence.
This commit is contained in:
Eric Coissac committed 2026-08-16 21:51:02 +02:00
1 parent 50f4820cb9
commit 3ba26b3dc1
14 files changed
+279 -24

No files matched your search

+1 -1
View File
@@ -22,7 +22,7 @@ mod sparse;
pub use builder::PersistentBitMatrixBuilder;
pub use packed::pack_bit_matrix;
pub use persistent::PersistentBitMatrix;
pub use sparse::{PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
pub use sparse::{PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_sparse_bit_matrix};
pub(crate) use pairwise::{pairwise_matrix, pairwise2_matrix};
+31 -3
View File
@@ -5,7 +5,7 @@
//! used by any production code path yet — a new type, not a replacement.
//!
//! On-disk layout (a directory, mirroring [`super::PersistentBitMatrix`]'s
//! own directory-of-files convention): `meta.json` plus four components —
//! own directory-of-files convention): `sparse_meta.json` plus four components —
//! `is_multi.prsb` (rank-capable flag: singleton row vs. multi-genome
//! row), `singleton.pfiv` (genome index, one entry per singleton row,
//! `ceil(log2(n_cols))` bits each), `multi.pfiv` (`dict_id`, one entry per
@@ -33,13 +33,14 @@ use crate::meta::field;
use crate::rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
use super::PersistentBitMatrix;
use super::packed::PackedBitMatrix;
fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") }
fn singleton_path(dir: &Path) -> PathBuf { dir.join("singleton.pfiv") }
fn multi_path(dir: &Path) -> PathBuf { dir.join("multi.pfiv") }
fn dict_offsets_base(dir: &Path) -> PathBuf { dir.join("dict_offsets") }
fn dict_values_path(dir: &Path) -> PathBuf { dir.join("dict_values.bin") }
fn meta_path(dir: &Path) -> PathBuf { dir.join("meta.json") }
fn meta_path(dir: &Path) -> PathBuf { dir.join("sparse_meta.json") }
struct SparseMeta {
n: usize,
@@ -53,7 +54,7 @@ impl SparseMeta {
fn load(dir: &Path) -> io::Result<Self> {
let s = fs::read_to_string(meta_path(dir))?;
let get = |name: &str| {
field(&s, name).ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, format!("bad meta.json: missing {name}")))
field(&s, name).ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, format!("bad sparse_meta.json: missing {name}")))
};
Ok(Self {
n: get("n")?,
@@ -370,3 +371,30 @@ impl PersistentSparseBitMatrixBuilder {
Ok(builder)
}
}
/// `pack --sparse`'s entry point: converts a presence directory into the
/// sparse on-disk format in place, mirroring [`super::pack_bit_matrix`]'s
/// convention (old-format files removed only after the new format is
/// fully written, so a crash mid-conversion leaves the previous, still
/// valid format rather than a half-written one). Idempotent — does
/// nothing if `is_multi.prsb` already exists. Packs to dense
/// (`matrix.pbmx`) first via [`super::pack_bit_matrix`] if that hasn't
/// happened yet, since the dense→sparse transpose (`build_from_dense`)
/// needs random row access, which only the packed/columnar dense forms
/// give — not a further reason to keep the dense file around afterward.
pub fn pack_sparse_bit_matrix(dir: &Path) -> io::Result<()> {
if is_multi_path(dir).exists() {
return Ok(());
}
let packed_path = dir.join("matrix.pbmx");
if !packed_path.exists() {
super::pack_bit_matrix(dir)?;
}
let dense = PersistentBitMatrix::Packed(PackedBitMatrix::open(&packed_path)?);
PersistentSparseBitMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
drop(dense);
fs::remove_file(&packed_path)?;
Ok(())
}
+1 -1
View File
@@ -19,7 +19,7 @@ pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
pub use eliasfano::{EliasFano, EliasFanoBuilder};
pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_bit_matrix};
pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix};
pub use builder::PersistentCompactIntVecBuilder;
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
+8
View File
@@ -26,6 +26,14 @@ pub trait BinaryMatrix {
fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<bool>]);
/// Per-genome k-mer totals.
fn count_ones(&self) -> Array1<u64>;
/// Allocating variant of [`fill_sub_matrix`](Self::fill_sub_matrix) —
/// provided once here so implementers only need the fill-in-place form.
fn sub_matrix(&self, slots: &[usize]) -> Vec<Vec<bool>> {
let mut out: Vec<Vec<bool>> = (0..self.n_cols()).map(|_| Vec::new()).collect();
self.fill_sub_matrix(slots, &mut out);
out
}
}
/// Convert a Jaccard distance matrix (`1 - J`) into a Mash distance matrix, per