Push zunrplorkwkt #70

Merged
coissac merged 93 commits from push-zunrplorkwkt into main 2026-08-28 23:15:38 +00:00
14 changed files with 779 additions and 369 deletions
Showing only changes of commit 4b6005962e - Show all commits
@@ -13,20 +13,30 @@ use std::error::Error;
use std::path::Path; use std::path::Path;
use std::time::Instant; use std::time::Instant;
use obicompactvec::{PersistentCompactIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder}; use obicompactvec::{
PersistentIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
};
fn main() -> Result<(), Box<dyn Error>> { fn main() -> Result<(), Box<dyn Error>> {
let layer_dir = std::env::args().nth(1).expect("usage: compare_sparse_dense_count <layer_dir>"); let layer_dir = std::env::args()
.nth(1)
.expect("usage: compare_sparse_dense_count <layer_dir>");
let layer_dir = Path::new(&layer_dir); let layer_dir = Path::new(&layer_dir);
let t0 = Instant::now(); let t0 = Instant::now();
let dense = PersistentCompactIntMatrix::open(layer_dir)?; let dense = PersistentIntMatrix::open(layer_dir)?;
println!("dense ouverte en {:?} ({} lignes x {} colonnes)", t0.elapsed(), dense.n(), dense.n_cols()); println!(
"dense ouverte en {:?} ({} lignes x {} colonnes)",
t0.elapsed(),
dense.n(),
dense.n_cols()
);
let out_dir = tempfile::tempdir()?; let out_dir = tempfile::tempdir()?;
let t0 = Instant::now(); let t0 = Instant::now();
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?.finish()?; let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?
.finish()?;
println!("build_from_dense: {:?}", t0.elapsed()); println!("build_from_dense: {:?}", t0.elapsed());
compare(&dense, &sparse)?; compare(&dense, &sparse)?;
@@ -36,7 +46,10 @@ fn main() -> Result<(), Box<dyn Error>> {
Ok(()) Ok(())
} }
fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactIntMatrix) -> Result<(), Box<dyn Error>> { fn compare(
dense: &PersistentIntMatrix,
sparse: &PersistentSparseCompactIntMatrix,
) -> Result<(), Box<dyn Error>> {
let n = dense.n(); let n = dense.n();
let n_cols = dense.n_cols(); let n_cols = dense.n_cols();
assert_eq!(n, sparse.n(), "n mismatch"); assert_eq!(n, sparse.n(), "n mismatch");
@@ -63,7 +76,10 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI
} }
} }
} }
println!("comparaison case-a-case: {:?} ({total_cells} cellules)", t0.elapsed()); println!(
"comparaison case-a-case: {:?} ({total_cells} cellules)",
t0.elapsed()
);
if let Some((slot, col, d, s)) = first_mismatch { if let Some((slot, col, d, s)) = first_mismatch {
eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}"); eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}");
@@ -78,7 +94,7 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI
/// for what's actually stored) — sparsity, singleton-row fraction, and the /// for what's actually stored) — sparsity, singleton-row fraction, and the
/// value-magnitude buckets this crate's overflow encoding is tuned around /// value-magnitude buckets this crate's overflow encoding is tuned around
/// (< 127, < 255, >= 255). /// (< 127, < 255, >= 255).
fn content_stats(dense: &PersistentCompactIntMatrix) { fn content_stats(dense: &PersistentIntMatrix) {
let n = dense.n(); let n = dense.n();
let n_cols = dense.n_cols(); let n_cols = dense.n_cols();
let total_cells = n as u64 * n_cols as u64; let total_cells = n as u64 * n_cols as u64;
@@ -128,7 +144,10 @@ fn content_stats(dense: &PersistentCompactIntMatrix) {
100.0 * singleton_rows as f64 / n as f64 100.0 * singleton_rows as f64 / n as f64
); );
if nonzero_cells > 0 { if nonzero_cells > 0 {
println!("valeur moyenne (non-nulles): {:.2}", sum as f64 / nonzero_cells as f64); println!(
"valeur moyenne (non-nulles): {:.2}",
sum as f64 / nonzero_cells as f64
);
println!("valeur max: {max_value}"); println!("valeur max: {max_value}");
println!( println!(
"valeurs < 127: {under_127} ({:.3}% des non-nulles)", "valeurs < 127: {under_127} ({:.3}% des non-nulles)",
@@ -162,7 +181,7 @@ fn dir_size(dir: &Path) -> std::io::Result<u64> {
fn compaction_stats( fn compaction_stats(
dense_layer_dir: &Path, dense_layer_dir: &Path,
sparse_dir: &Path, sparse_dir: &Path,
dense: &PersistentCompactIntMatrix, dense: &PersistentIntMatrix,
_sparse: &PersistentSparseCompactIntMatrix, _sparse: &PersistentSparseCompactIntMatrix,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
let dense_size = dir_size(&dense_layer_dir.join("counts"))?; let dense_size = dir_size(&dense_layer_dir.join("counts"))?;
+254 -77
View File
@@ -5,17 +5,19 @@ use std::path::{Path, PathBuf};
use memmap2::Mmap; use memmap2::Mmap;
use ndarray::{Array1, Array2}; use ndarray::{Array1, Array2};
use crate::bitmatrix::{fill_row_generic, par_col_reduce, pairwise2_matrix, pairwise_matrix, row_generic}; use crate::bitmatrix::{
fill_row_generic, pairwise_matrix, pairwise2_matrix, par_col_reduce, row_generic,
};
use crate::builder::PersistentCompactIntVecBuilder; use crate::builder::PersistentCompactIntVecBuilder;
use crate::colgroup::{chunked_presence_count, ColGroup, MatrixGroupOps}; use crate::colgroup::{ColGroup, MatrixGroupOps, chunked_presence_count};
use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE}; use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE};
use crate::meta::MatrixMeta; use crate::meta::MatrixMeta;
use crate::reader::PersistentCompactIntVec; use crate::reader::PersistentCompactIntVec;
use crate::sparse_intmatrix::{is_present as sparse_is_present, PersistentSparseCompactIntMatrix}; use crate::sparse_intmatrix::{PersistentSparseCompactIntMatrix, is_present as sparse_is_present};
use crate::storage_kind::StorageKind; use crate::storage_kind::StorageKind;
use crate::tempbitvec::{TempBitVec, TempBitVecBuilder}; use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder}; use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
use crate::views::{nonzero_triples, IntSliceView}; use crate::views::{IntSliceView, nonzero_triples};
pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf { pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf {
dir.join(format!("col_{col:06}.pciv")) dir.join(format!("col_{col:06}.pciv"))
@@ -38,11 +40,17 @@ impl ColumnarCompactIntMatrix {
} }
#[inline] #[inline]
pub(crate) fn n(&self) -> usize { self.n } pub(crate) fn n(&self) -> usize {
self.n
}
#[inline] #[inline]
pub(crate) fn n_cols(&self) -> usize { self.cols.len() } pub(crate) fn n_cols(&self) -> usize {
self.cols.len()
}
#[inline] #[inline]
pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec { &self.cols[c] } pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec {
&self.cols[c]
}
#[inline] #[inline]
pub(crate) fn row(&self, slot: usize) -> Box<[u32]> { pub(crate) fn row(&self, slot: usize) -> Box<[u32]> {
@@ -63,34 +71,64 @@ impl ColumnarCompactIntMatrix {
} }
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> { pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_bray_dist(self.col(j))) pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_bray_dist(self.col(j))
})
} }
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> { pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_euclidean_dist(self.col(j))) pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_euclidean_dist(self.col(j))
})
} }
pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) { pub(crate) fn partial_threshold_jaccard_dist_matrix(
pairwise2_matrix(self.n_cols(), |i, j| self.col(i).partial_threshold_jaccard_dist(self.col(j), threshold)) &self,
threshold: u32,
) -> (Array2<u64>, Array2<u64>) {
pairwise2_matrix(self.n_cols(), |i, j| {
self.col(i)
.partial_threshold_jaccard_dist(self.col(j), threshold)
})
} }
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> { pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| { pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_relfreq_bray_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64) self.col(i).partial_relfreq_bray_dist(
self.col(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
}) })
} }
pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> { pub(crate) fn partial_relfreq_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| { pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_relfreq_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64) self.col(i).partial_relfreq_euclidean_dist(
self.col(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
}) })
} }
pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> { pub(crate) fn partial_hellinger_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| { pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_hellinger_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64) self.col(i).partial_hellinger_euclidean_dist(
self.col(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
}) })
} }
pub(crate) fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> { pub(crate) fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> {
let mut meta = MatrixMeta::load(dir)?; let mut meta = MatrixMeta::load(dir)?;
let mut b = PersistentCompactIntVecBuilder::new(meta.n, &col_path(dir, meta.n_cols))?; let mut b = PersistentCompactIntVecBuilder::new(meta.n, &col_path(dir, meta.n_cols))?;
for slot in 0..meta.n { b.set(slot, value_of(slot)); } for slot in 0..meta.n {
b.set(slot, value_of(slot));
}
b.close()?; b.close()?;
meta.n_cols += 1; meta.n_cols += 1;
meta.save(dir) meta.save(dir)
@@ -119,7 +157,10 @@ impl PackedCompactIntMatrix {
pub(crate) fn open(path: &Path) -> io::Result<Self> { pub(crate) fn open(path: &Path) -> io::Result<Self> {
let mmap = unsafe { Mmap::map(&File::open(path)?)? }; let mmap = unsafe { Mmap::map(&File::open(path)?)? };
if mmap.len() < PCMX_HEADER { if mmap.len() < PCMX_HEADER {
return Err(io::Error::new(io::ErrorKind::InvalidData, "PCMX file too short")); return Err(io::Error::new(
io::ErrorKind::InvalidData,
"PCMX file too short",
));
} }
if &mmap[0..4] != &PCMX_MAGIC { if &mmap[0..4] != &PCMX_MAGIC {
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PCMX magic")); return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PCMX magic"));
@@ -130,32 +171,51 @@ impl PackedCompactIntMatrix {
let mut columns = Vec::with_capacity(n_cols); let mut columns = Vec::with_capacity(n_cols);
for c in 0..n_cols { for c in 0..n_cols {
let off_pos = PCMX_HEADER + c * 8; let off_pos = PCMX_HEADER + c * 8;
let col_base = u64::from_le_bytes(mmap[off_pos..off_pos+8].try_into().unwrap()) as usize; let col_base =
let n_ov = u64::from_le_bytes(mmap[col_base+16..col_base+24].try_into().unwrap()) as usize; u64::from_le_bytes(mmap[off_pos..off_pos + 8].try_into().unwrap()) as usize;
let n_pciv = u64::from_le_bytes(mmap[col_base+8..col_base+16].try_into().unwrap()) as usize; let n_ov =
u64::from_le_bytes(mmap[col_base + 16..col_base + 24].try_into().unwrap()) as usize;
let n_pciv =
u64::from_le_bytes(mmap[col_base + 8..col_base + 16].try_into().unwrap()) as usize;
let primary_start = col_base + HEADER_SIZE; let primary_start = col_base + HEADER_SIZE;
let data_offset = primary_start + n_pciv; let data_offset = primary_start + n_pciv;
columns.push(ColInfo { primary_start, data_offset, n_overflow: n_ov }); columns.push(ColInfo {
primary_start,
data_offset,
n_overflow: n_ov,
});
} }
Ok(Self { mmap, n_rows, n_cols, columns }) Ok(Self {
mmap,
n_rows,
n_cols,
columns,
})
} }
#[inline] #[inline]
pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> { pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> {
let ci = &self.columns[c]; let ci = &self.columns[c];
let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows]; let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
let overflow_raw = &self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE]; let overflow_raw =
&self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE];
IntSliceView::new(primary, overflow_raw, ci.n_overflow, self.n_rows) IntSliceView::new(primary, overflow_raw, ci.n_overflow, self.n_rows)
} }
pub(crate) fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentCompactIntVecBuilder> { pub(crate) fn col_persist(
&self,
c: usize,
path: &Path,
) -> io::Result<PersistentCompactIntVecBuilder> {
let view = self.col_view(c); let view = self.col_view(c);
let overflow: std::collections::HashMap<usize, u32> = view.overflow_entries().collect(); let overflow: std::collections::HashMap<usize, u32> = view.overflow_entries().collect();
PersistentCompactIntVecBuilder::from_raw_primary(view.primary_bytes(), overflow, path) PersistentCompactIntVecBuilder::from_raw_primary(view.primary_bytes(), overflow, path)
} }
#[inline] #[inline]
pub(crate) fn get(&self, col: usize, slot: usize) -> u32 { self.col_view(col).get(slot) } pub(crate) fn get(&self, col: usize, slot: usize) -> u32 {
self.col_view(col).get(slot)
}
#[inline] #[inline]
pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) { pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
@@ -180,27 +240,55 @@ impl PackedCompactIntMatrix {
// via `PersistentCompactIntVec`. No formula is re-derived here. // via `PersistentCompactIntVec`. No formula is re-derived here.
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> { pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_bray_dist(self.col_view(j))) pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_bray_dist(self.col_view(j))
})
} }
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> { pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_euclidean_dist(self.col_view(j))) pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_euclidean_dist(self.col_view(j))
})
} }
pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, t: u32) -> (Array2<u64>, Array2<u64>) { pub(crate) fn partial_threshold_jaccard_dist_matrix(
pairwise2_matrix(self.n_cols, |i, j| self.col_view(i).partial_threshold_jaccard_dist(self.col_view(j), t)) &self,
t: u32,
) -> (Array2<u64>, Array2<u64>) {
pairwise2_matrix(self.n_cols, |i, j| {
self.col_view(i)
.partial_threshold_jaccard_dist(self.col_view(j), t)
})
} }
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> { pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| { pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_relfreq_bray_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64) self.col_view(i).partial_relfreq_bray_dist(
self.col_view(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
}) })
} }
pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> { pub(crate) fn partial_relfreq_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| { pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_relfreq_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64) self.col_view(i).partial_relfreq_euclidean_dist(
self.col_view(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
}) })
} }
pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> { pub(crate) fn partial_hellinger_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| { pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_hellinger_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64) self.col_view(i).partial_hellinger_euclidean_dist(
self.col_view(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
}) })
} }
} }
@@ -238,7 +326,9 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
// the columnar files are newer and must be (re-)packed, overwriting the // the columnar files are newer and must be (re-)packed, overwriting the
// stale one — never silently discarded as "leftover cleanup". // stale one — never silently discarded as "leftover cleanup".
if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) { if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); } for c in 0..meta.n_cols {
let _ = fs::remove_file(col_path(dir, c));
}
let _ = fs::remove_file(dir.join("meta.json")); let _ = fs::remove_file(dir.join("meta.json"));
return Ok(()); return Ok(());
} }
@@ -250,32 +340,41 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
let header_size = (PCMX_HEADER + n_cols * 8) as u64; let header_size = (PCMX_HEADER + n_cols * 8) as u64;
let mut col_offset = header_size; let mut col_offset = header_size;
let mut offsets = Vec::with_capacity(n_cols); let mut offsets = Vec::with_capacity(n_cols);
for &size in &col_sizes { offsets.push(col_offset); col_offset += size; } for &size in &col_sizes {
offsets.push(col_offset);
col_offset += size;
}
let tmp_path = dir.join("matrix.pcmx.tmp"); let tmp_path = dir.join("matrix.pcmx.tmp");
let mut out = BufWriter::new(File::create(&tmp_path)?); let mut out = BufWriter::new(File::create(&tmp_path)?);
out.write_all(&PCMX_MAGIC)?; out.write_all(&PCMX_MAGIC)?;
out.write_all(&[0u8; 4])?; out.write_all(&[0u8; 4])?;
out.write_all(&(meta.n as u64).to_le_bytes())?; out.write_all(&(meta.n as u64).to_le_bytes())?;
out.write_all(&(n_cols as u64).to_le_bytes())?; out.write_all(&(n_cols as u64).to_le_bytes())?;
for &off in &offsets { out.write_all(&off.to_le_bytes())?; } for &off in &offsets {
for c in 0..n_cols { io::copy(&mut File::open(col_path(dir, c))?, &mut out)?; } out.write_all(&off.to_le_bytes())?;
}
for c in 0..n_cols {
io::copy(&mut File::open(col_path(dir, c))?, &mut out)?;
}
out.flush()?; out.flush()?;
drop(out); drop(out);
fs::rename(&tmp_path, &packed_path)?; fs::rename(&tmp_path, &packed_path)?;
for c in 0..n_cols { fs::remove_file(col_path(dir, c))?; } for c in 0..n_cols {
fs::remove_file(col_path(dir, c))?;
}
fs::remove_file(dir.join("meta.json"))?; fs::remove_file(dir.join("meta.json"))?;
Ok(()) Ok(())
} }
// ── PersistentCompactIntMatrix — public enum ────────────────────────────────── // ── PersistentCompactIntMatrix — public enum ──────────────────────────────────
pub enum PersistentCompactIntMatrix { pub enum PersistentIntMatrix {
Columnar(ColumnarCompactIntMatrix), Columnar(ColumnarCompactIntMatrix),
Packed(PackedCompactIntMatrix), Packed(PackedCompactIntMatrix),
Sparse(PersistentSparseCompactIntMatrix), Sparse(PersistentSparseCompactIntMatrix),
} }
impl PersistentCompactIntMatrix { impl PersistentIntMatrix {
/// Checks (in order): `counts/matrix.pcmx` → Packed; `counts/meta.json` /// Checks (in order): `counts/matrix.pcmx` → Packed; `counts/meta.json`
/// → Columnar; `counts/singleton_values.pciv` → Sparse. Mirrors /// → Columnar; `counts/singleton_values.pciv` → Sparse. Mirrors
/// `PersistentBitMatrix::open`'s own Packed → Columnar → Sparse /// `PersistentBitMatrix::open`'s own Packed → Columnar → Sparse
@@ -283,17 +382,24 @@ impl PersistentCompactIntMatrix {
pub fn open(layer_dir: &Path) -> io::Result<Self> { pub fn open(layer_dir: &Path) -> io::Result<Self> {
let counts_dir = layer_dir.join("counts"); let counts_dir = layer_dir.join("counts");
if counts_dir.join("matrix.pcmx").exists() { if counts_dir.join("matrix.pcmx").exists() {
return Ok(Self::Packed(PackedCompactIntMatrix::open(&counts_dir.join("matrix.pcmx"))?)); return Ok(Self::Packed(PackedCompactIntMatrix::open(
&counts_dir.join("matrix.pcmx"),
)?));
} }
if MatrixMeta::load(&counts_dir).is_ok() { if MatrixMeta::load(&counts_dir).is_ok() {
return Ok(Self::Columnar(ColumnarCompactIntMatrix::open(&counts_dir)?)); return Ok(Self::Columnar(ColumnarCompactIntMatrix::open(&counts_dir)?));
} }
if sparse_is_present(&counts_dir) { if sparse_is_present(&counts_dir) {
return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open(&counts_dir)?)); return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open(
&counts_dir,
)?));
} }
Err(io::Error::new( Err(io::Error::new(
io::ErrorKind::NotFound, io::ErrorKind::NotFound,
format!("no count matrix found in {} — run 'obikmer upgrade'", layer_dir.display()), format!(
"no count matrix found in {} — run 'obikmer upgrade'",
layer_dir.display()
),
)) ))
} }
@@ -330,11 +436,19 @@ impl PersistentCompactIntMatrix {
#[inline] #[inline]
pub fn n(&self) -> usize { pub fn n(&self) -> usize {
match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows, Self::Sparse(m) => m.n() } match self {
Self::Columnar(m) => m.n(),
Self::Packed(m) => m.n_rows,
Self::Sparse(m) => m.n(),
}
} }
#[inline] #[inline]
pub fn n_cols(&self) -> usize { pub fn n_cols(&self) -> usize {
match self { Self::Columnar(m) => m.n_cols(), Self::Packed(m) => m.n_cols, Self::Sparse(m) => m.n_cols() } match self {
Self::Columnar(m) => m.n_cols(),
Self::Packed(m) => m.n_cols,
Self::Sparse(m) => m.n_cols(),
}
} }
#[inline] #[inline]
@@ -350,7 +464,9 @@ impl PersistentCompactIntMatrix {
match self { match self {
Self::Columnar(m) => m.col(c).view(), Self::Columnar(m) => m.col(c).view(),
Self::Packed(m) => m.col_view(c), Self::Packed(m) => m.col_view(c),
Self::Sparse(_) => panic!("col_view() not available on Sparse PersistentCompactIntMatrix"), Self::Sparse(_) => {
panic!("col_view() not available on Sparse PersistentCompactIntMatrix")
}
} }
} }
@@ -358,14 +474,20 @@ impl PersistentCompactIntMatrix {
match self { match self {
Self::Columnar(m) => PersistentCompactIntVecBuilder::build_from(m.col(c), path), Self::Columnar(m) => PersistentCompactIntVecBuilder::build_from(m.col(c), path),
Self::Packed(m) => m.col_persist(c, path), Self::Packed(m) => m.col_persist(c, path),
Self::Sparse(_) => Err(io::Error::new(io::ErrorKind::Unsupported, Self::Sparse(_) => Err(io::Error::new(
"col_persist not available on Sparse PersistentCompactIntMatrix")), io::ErrorKind::Unsupported,
"col_persist not available on Sparse PersistentCompactIntMatrix",
)),
} }
} }
#[inline] #[inline]
pub fn row(&self, slot: usize) -> Box<[u32]> { pub fn row(&self, slot: usize) -> Box<[u32]> {
match self { Self::Columnar(m) => m.row(slot), Self::Packed(m) => m.row(slot), Self::Sparse(m) => m.row(slot) } match self {
Self::Columnar(m) => m.row(slot),
Self::Packed(m) => m.row(slot),
Self::Sparse(m) => m.row(slot),
}
} }
#[inline] #[inline]
pub fn fill_row(&self, slot: usize, buf: &mut [u32]) { pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
@@ -418,18 +540,28 @@ impl PersistentCompactIntMatrix {
/// `Columnar`/`Packed` reuse the shared sorted-slot batching in /// `Columnar`/`Packed` reuse the shared sorted-slot batching in
/// `views::nonzero_triples`. Boxed, not `impl Iterator`, because the /// `views::nonzero_triples`. Boxed, not `impl Iterator`, because the
/// match arms are genuinely different concrete types. /// match arms are genuinely different concrete types.
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> { pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
match self { match self {
Self::Sparse(m) => Box::new(m.nonzero_iter(slots)), Self::Sparse(m) => Box::new(m.nonzero_iter(slots)),
Self::Columnar(_) | Self::Packed(_) => { Self::Columnar(_) | Self::Packed(_) => Box::new(nonzero_triples(
Box::new(nonzero_triples(slots, self.n_cols(), |c| self.col_view(c), false)) slots,
} self.n_cols(),
|c| self.col_view(c),
false,
)),
} }
} }
#[inline] #[inline]
pub fn sum(&self) -> Array1<u64> { pub fn sum(&self) -> Array1<u64> {
match self { Self::Columnar(m) => m.sum(), Self::Packed(m) => m.sum(), Self::Sparse(m) => m.sum() } match self {
Self::Columnar(m) => m.sum(),
Self::Packed(m) => m.sum(),
Self::Sparse(m) => m.sum(),
}
} }
#[inline] #[inline]
pub fn count_nonzero(&self) -> Array1<u64> { pub fn count_nonzero(&self) -> Array1<u64> {
@@ -456,7 +588,10 @@ impl PersistentCompactIntMatrix {
} }
} }
#[inline] #[inline]
pub fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) { pub fn partial_threshold_jaccard_dist_matrix(
&self,
threshold: u32,
) -> (Array2<u64>, Array2<u64>) {
match self { match self {
Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold), Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold), Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
@@ -497,26 +632,42 @@ impl PersistentCompactIntMatrix {
use crate::traits::{ColumnWeights, CountPartials}; use crate::traits::{ColumnWeights, CountPartials};
impl ColumnWeights for PersistentCompactIntMatrix { impl ColumnWeights for PersistentIntMatrix {
#[inline] #[inline]
fn col_weights(&self) -> Array1<u64> { self.sum() } fn col_weights(&self) -> Array1<u64> {
self.sum()
}
#[inline] #[inline]
fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() } fn partial_kmer_counts(&self) -> Array1<u64> {
self.count_nonzero()
}
} }
impl CountPartials for PersistentCompactIntMatrix { impl CountPartials for PersistentIntMatrix {
#[inline] #[inline]
fn partial_bray(&self) -> Array2<u64> { self.partial_bray_dist_matrix() } fn partial_bray(&self) -> Array2<u64> {
self.partial_bray_dist_matrix()
}
#[inline] #[inline]
fn partial_euclidean(&self) -> Array2<f64> { self.partial_euclidean_dist_matrix() } fn partial_euclidean(&self) -> Array2<f64> {
self.partial_euclidean_dist_matrix()
}
#[inline] #[inline]
fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) { self.partial_threshold_jaccard_dist_matrix(t) } fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) {
self.partial_threshold_jaccard_dist_matrix(t)
}
#[inline] #[inline]
fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_bray_dist_matrix(g) } fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> {
self.partial_relfreq_bray_dist_matrix(g)
}
#[inline] #[inline]
fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_euclidean_dist_matrix(g) } fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> {
self.partial_relfreq_euclidean_dist_matrix(g)
}
#[inline] #[inline]
fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_hellinger_euclidean_dist_matrix(g) } fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> {
self.partial_hellinger_euclidean_dist_matrix(g)
}
} }
// ── Builder ─────────────────────────────────────────────────────────────────── // ── Builder ───────────────────────────────────────────────────────────────────
@@ -530,7 +681,11 @@ pub struct PersistentCompactIntMatrixBuilder {
impl PersistentCompactIntMatrixBuilder { impl PersistentCompactIntMatrixBuilder {
pub fn new(n: usize, dir: &Path) -> io::Result<Self> { pub fn new(n: usize, dir: &Path) -> io::Result<Self> {
fs::create_dir_all(dir)?; fs::create_dir_all(dir)?;
Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 }) Ok(Self {
dir: dir.to_path_buf(),
n,
n_cols: 0,
})
} }
/// Resume appending columns to a Columnar matrix directory already /// Resume appending columns to a Columnar matrix directory already
@@ -541,13 +696,21 @@ impl PersistentCompactIntMatrixBuilder {
/// on-disk column naming or `MatrixMeta` themselves. /// on-disk column naming or `MatrixMeta` themselves.
pub fn resume(dir: &Path) -> io::Result<Self> { pub fn resume(dir: &Path) -> io::Result<Self> {
let meta = MatrixMeta::load(dir)?; let meta = MatrixMeta::load(dir)?;
Ok(Self { dir: dir.to_path_buf(), n: meta.n, n_cols: meta.n_cols }) Ok(Self {
dir: dir.to_path_buf(),
n: meta.n,
n_cols: meta.n_cols,
})
} }
#[inline] #[inline]
pub fn n(&self) -> usize { self.n } pub fn n(&self) -> usize {
self.n
}
#[inline] #[inline]
pub fn n_cols(&self) -> usize { self.n_cols } pub fn n_cols(&self) -> usize {
self.n_cols
}
pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> { pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> {
let path = col_path(&self.dir, self.n_cols); let path = col_path(&self.dir, self.n_cols);
self.n_cols += 1; self.n_cols += 1;
@@ -569,14 +732,22 @@ impl PersistentCompactIntMatrixBuilder {
} }
pub fn close(self) -> io::Result<()> { pub fn close(self) -> io::Result<()> {
MatrixMeta { n: self.n, n_cols: self.n_cols }.save(&self.dir) MatrixMeta {
n: self.n,
n_cols: self.n_cols,
}
.save(&self.dir)
} }
} }
// ── MatrixGroupOps ──────────────────────────────────────────────────────────── // ── MatrixGroupOps ────────────────────────────────────────────────────────────
impl MatrixGroupOps for PersistentCompactIntMatrix { impl MatrixGroupOps for PersistentIntMatrix {
fn partial_group_presence_count(&self, g: &ColGroup, threshold: u32) -> io::Result<TempCompactIntVec> { fn partial_group_presence_count(
&self,
g: &ColGroup,
threshold: u32,
) -> io::Result<TempCompactIntVec> {
chunked_presence_count(self.n(), &g.indices, |b, c| { chunked_presence_count(self.n(), &g.indices, |b, c| {
b.inc_predicate_fast(self.col_view(c), |v| v >= threshold) b.inc_predicate_fast(self.col_view(c), |v| v >= threshold)
}) })
@@ -585,7 +756,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> { fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
let n = self.n(); let n = self.n();
let mut result = TempCompactIntVecBuilder::new(n)?; let mut result = TempCompactIntVecBuilder::new(n)?;
for &c in &g.indices { result.add(self.col_view(c)); } for &c in &g.indices {
result.add(self.col_view(c));
}
result.freeze() result.freeze()
} }
@@ -603,7 +776,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
let mut result = TempCompactIntVecBuilder::new(n)?; let mut result = TempCompactIntVecBuilder::new(n)?;
if let Some((&first, rest)) = g.indices.split_first() { if let Some((&first, rest)) = g.indices.split_first() {
result.add(self.col_view(first)); result.add(self.col_view(first));
for &c in rest { result.min(self.col_view(c)); } for &c in rest {
result.min(self.col_view(c));
}
} }
result.freeze() result.freeze()
} }
@@ -611,7 +786,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> { fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
let n = self.n(); let n = self.n();
let mut result = TempCompactIntVecBuilder::new(n)?; let mut result = TempCompactIntVecBuilder::new(n)?;
for &c in &g.indices { result.max(self.col_view(c)); } for &c in &g.indices {
result.max(self.col_view(c));
}
result.freeze() result.freeze()
} }
} }
+20 -11
View File
@@ -1,41 +1,50 @@
mod bitvec;
mod bitmatrix; mod bitmatrix;
mod bitvec;
mod builder; mod builder;
mod colgroup; mod colgroup;
mod eliasfano; mod eliasfano;
mod fixedintvec; mod fixedintvec;
mod format; mod format;
mod rankselect;
mod intmatrix; mod intmatrix;
mod layer_meta; mod layer_meta;
mod matrix_builder; mod matrix_builder;
mod meta; mod meta;
mod mmap_file; mod mmap_file;
mod rankselect;
mod reader; mod reader;
mod sparse_intmatrix; mod sparse_intmatrix;
mod storage_kind; mod storage_kind;
mod tempbitvec; mod tempbitvec;
mod tempintvec; mod tempintvec;
mod views;
pub mod traits; pub mod traits;
mod views;
pub use bitmatrix::{
PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix,
PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix,
};
pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder}; pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
pub use eliasfano::{EliasFano, EliasFanoBuilder};
pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix};
pub use builder::PersistentCompactIntVecBuilder; pub use builder::PersistentCompactIntVecBuilder;
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask}; pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix}; pub use eliasfano::{EliasFano, EliasFanoBuilder};
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
pub use intmatrix::{
PersistentCompactIntMatrixBuilder, PersistentIntMatrix, pack_compact_int_matrix,
};
pub use layer_meta::LayerMeta; pub use layer_meta::LayerMeta;
pub use matrix_builder::{ColBuilder, MatrixBuilder}; pub use matrix_builder::{ColBuilder, MatrixBuilder};
pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter}; pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
pub use sparse_intmatrix::{PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder, pack_sparse_compact_int_matrix}; pub use reader::{Iter as CompactIntVecIter, PersistentCompactIntVec};
pub use sparse_intmatrix::{
PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
pack_sparse_compact_int_matrix,
};
pub use storage_kind::StorageKind; pub use storage_kind::StorageKind;
pub use tempbitvec::{TempBitVec, TempBitVecBuilder}; pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder}; pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials}; pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials};
pub use views::{BitSliceView, BitSliceIter, IntSliceView, IntSliceViewIter}; pub use views::{BitSliceIter, BitSliceView, IntSliceView, IntSliceViewIter};
#[cfg(test)] #[cfg(test)]
#[path = "tests/mod.rs"] #[path = "tests/mod.rs"]
+88 -28
View File
@@ -35,11 +35,17 @@ use crate::builder::PersistentCompactIntVecBuilder;
use crate::eliasfano::{EliasFano, EliasFanoBuilder}; use crate::eliasfano::{EliasFano, EliasFanoBuilder};
use crate::reader::PersistentCompactIntVec; use crate::reader::PersistentCompactIntVec;
use crate::traits::{BitPartials, ColumnWeights, CountPartials}; use crate::traits::{BitPartials, ColumnWeights, CountPartials};
use crate::{PersistentCompactIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder}; use crate::{PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
fn singleton_values_path(dir: &Path) -> PathBuf { dir.join("singleton_values.pciv") } fn singleton_values_path(dir: &Path) -> PathBuf {
fn multi_values_path(dir: &Path) -> PathBuf { dir.join("multi_values.pciv") } dir.join("singleton_values.pciv")
fn multi_offsets_base(dir: &Path) -> PathBuf { dir.join("multi_offsets") } }
fn multi_values_path(dir: &Path) -> PathBuf {
dir.join("multi_values.pciv")
}
fn multi_offsets_base(dir: &Path) -> PathBuf {
dir.join("multi_offsets")
}
/// Lightweight disk probe: true iff a sparse count matrix has already been /// Lightweight disk probe: true iff a sparse count matrix has already been
/// written at `dir` — the marker [`PersistentCompactIntMatrix::open`]/ /// written at `dir` — the marker [`PersistentCompactIntMatrix::open`]/
@@ -73,9 +79,13 @@ impl PersistentSparseCompactIntMatrix {
} }
#[inline] #[inline]
pub fn n(&self) -> usize { self.support.n() } pub fn n(&self) -> usize {
self.support.n()
}
#[inline] #[inline]
pub fn n_cols(&self) -> usize { self.support.n_cols() } pub fn n_cols(&self) -> usize {
self.support.n_cols()
}
/// Calls `f(col, value)` once per non-zero column at `slot`, in /// Calls `f(col, value)` once per non-zero column at `slot`, in
/// ascending column order — the shared decode branch behind every /// ascending column order — the shared decode branch behind every
@@ -104,7 +114,11 @@ impl PersistentSparseCompactIntMatrix {
/// absent). /// absent).
pub fn get(&self, c: usize, slot: usize) -> u32 { pub fn get(&self, c: usize, slot: usize) -> u32 {
let mut found = 0u32; let mut found = 0u32;
self.for_each_cell_in_row(slot, |g, v| if g == c { found = v; }); self.for_each_cell_in_row(slot, |g, v| {
if g == c {
found = v;
}
});
found found
} }
@@ -138,13 +152,17 @@ impl PersistentSparseCompactIntMatrix {
/// Yields every `(idx into slots, col, value)` triple, row by row, in /// Yields every `(idx into slots, col, value)` triple, row by row, in
/// `slots` order — mirrors /// `slots` order — mirrors
/// [`PersistentSparseBitMatrix::nonzero_iter`]. /// [`PersistentSparseBitMatrix::nonzero_iter`].
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator<Item = (usize, usize, u32)> + 'a { pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> impl Iterator<Item = (usize, usize, u32)> + 'a {
let mut next_slot = 0usize; let mut next_slot = 0usize;
let mut cur_idx = 0usize; let mut cur_idx = 0usize;
let mut buf: Vec<(usize, u32)> = Vec::new(); let mut buf: Vec<(usize, u32)> = Vec::new();
let mut buf_pos = 0usize; let mut buf_pos = 0usize;
std::iter::from_fn(move || loop { std::iter::from_fn(move || {
loop {
if buf_pos < buf.len() { if buf_pos < buf.len() {
let (g, v) = buf[buf_pos]; let (g, v) = buf[buf_pos];
buf_pos += 1; buf_pos += 1;
@@ -159,6 +177,7 @@ impl PersistentSparseCompactIntMatrix {
buf.clear(); buf.clear();
self.for_each_cell_in_row(slot, |g, v| buf.push((g, v))); self.for_each_cell_in_row(slot, |g, v| buf.push((g, v)));
buf_pos = 0; buf_pos = 0;
}
}) })
} }
@@ -203,7 +222,11 @@ impl PersistentSparseCompactIntMatrix {
fn count_geq(&self, threshold: u32) -> Array1<u64> { fn count_geq(&self, threshold: u32) -> Array1<u64> {
let mut counts = vec![0u64; self.n_cols()]; let mut counts = vec![0u64; self.n_cols()];
for slot in 0..self.n() { for slot in 0..self.n() {
self.for_each_cell_in_row(slot, |g, v| if v >= threshold { counts[g] += 1; }); self.for_each_cell_in_row(slot, |g, v| {
if v >= threshold {
counts[g] += 1;
}
});
} }
Array1::from(counts) Array1::from(counts)
} }
@@ -261,9 +284,13 @@ impl PersistentSparseCompactIntMatrix {
impl ColumnWeights for PersistentSparseCompactIntMatrix { impl ColumnWeights for PersistentSparseCompactIntMatrix {
#[inline] #[inline]
fn col_weights(&self) -> Array1<u64> { self.sum() } fn col_weights(&self) -> Array1<u64> {
self.sum()
}
#[inline] #[inline]
fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() } fn partial_kmer_counts(&self) -> Array1<u64> {
self.count_nonzero()
}
} }
impl CountPartials for PersistentSparseCompactIntMatrix { impl CountPartials for PersistentSparseCompactIntMatrix {
@@ -278,7 +305,9 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
fn partial_bray(&self) -> Array2<u64> { fn partial_bray(&self) -> Array2<u64> {
let mut m = self.row_major_pairwise(|_, a, _, b| (a as u64).min(b as u64)); let mut m = self.row_major_pairwise(|_, a, _, b| (a as u64).min(b as u64));
let sum = self.sum(); let sum = self.sum();
for i in 0..self.n_cols() { m[[i, i]] = sum[i]; } for i in 0..self.n_cols() {
m[[i, i]] = sum[i];
}
m m
} }
@@ -329,7 +358,8 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
// left to patch. // left to patch.
return BitPartials::partial_jaccard(&self.support); return BitPartials::partial_jaccard(&self.support);
} }
let mut inter = self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64); let mut inter =
self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64);
let mut union = Array2::<u64>::zeros((n, n)); let mut union = Array2::<u64>::zeros((n, n));
for i in 0..n { for i in 0..n {
for j in 0..n { for j in 0..n {
@@ -363,7 +393,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
}); });
let sum = self.sum(); let sum = self.sum();
for i in 0..self.n_cols() { for i in 0..self.n_cols() {
m[[i, i]] = if global[i] > 0 { sum[i] as f64 / global[i] as f64 } else { 0.0 }; m[[i, i]] = if global[i] > 0 {
sum[i] as f64 / global[i] as f64
} else {
0.0
};
} }
m m
} }
@@ -382,9 +416,21 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
if i != j { if i != j {
let sa = global[i] as f64; let sa = global[i] as f64;
let sb = global[j] as f64; let sb = global[j] as f64;
let sq_a = if sa > 0.0 { sq[i] as f64 / (sa * sa) } else { 0.0 }; let sq_a = if sa > 0.0 {
let sq_b = if sb > 0.0 { sq[j] as f64 / (sb * sb) } else { 0.0 }; sq[i] as f64 / (sa * sa)
let cross = if sa > 0.0 && sb > 0.0 { dot[[i, j]] / (sa * sb) } else { 0.0 }; } else {
0.0
};
let sq_b = if sb > 0.0 {
sq[j] as f64 / (sb * sb)
} else {
0.0
};
let cross = if sa > 0.0 && sb > 0.0 {
dot[[i, j]] / (sa * sb)
} else {
0.0
};
m[[i, j]] = sq_a + sq_b - 2.0 * cross; m[[i, j]] = sq_a + sq_b - 2.0 * cross;
} }
} }
@@ -409,7 +455,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
let sb = global[j] as f64; let sb = global[j] as f64;
let pa_sum = if sa > 0.0 { sum[i] as f64 / sa } else { 0.0 }; let pa_sum = if sa > 0.0 { sum[i] as f64 / sa } else { 0.0 };
let pb_sum = if sb > 0.0 { sum[j] as f64 / sb } else { 0.0 }; let pb_sum = if sb > 0.0 { sum[j] as f64 / sb } else { 0.0 };
let cross = if sa > 0.0 && sb > 0.0 { sqrt_dot[[i, j]] / (sa * sb).sqrt() } else { 0.0 }; let cross = if sa > 0.0 && sb > 0.0 {
sqrt_dot[[i, j]] / (sa * sb).sqrt()
} else {
0.0
};
m[[i, j]] = pa_sum + pb_sum - 2.0 * cross; m[[i, j]] = pa_sum + pb_sum - 2.0 * cross;
} }
} }
@@ -461,7 +511,6 @@ impl PersistentSparseCompactIntMatrixBuilder {
self.support.push_row(cols); self.support.push_row(cols);
} }
/// Builds a sparse matrix from an already-built dense /// Builds a sparse matrix from an already-built dense
/// [`PersistentCompactIntMatrix`] (`Columnar` or `Packed`) — same /// [`PersistentCompactIntMatrix`] (`Columnar` or `Packed`) — same
/// batched-transpose shape as /// batched-transpose shape as
@@ -475,7 +524,7 @@ impl PersistentSparseCompactIntMatrixBuilder {
/// pairs are appended to its own buffer in strictly ascending column /// pairs are appended to its own buffer in strictly ascending column
/// order "for free", matching `push_row`'s ascending-`cols` contract /// order "for free", matching `push_row`'s ascending-`cols` contract
/// without an explicit sort. /// without an explicit sort.
pub fn build_from_dense(dense: &PersistentCompactIntMatrix, dir: &Path) -> io::Result<Self> { pub fn build_from_dense(dense: &PersistentIntMatrix, dir: &Path) -> io::Result<Self> {
let n = dense.n(); let n = dense.n();
let n_cols = dense.n_cols(); let n_cols = dense.n_cols();
let mut builder = Self::new(n, n_cols, dir)?; let mut builder = Self::new(n, n_cols, dir)?;
@@ -500,23 +549,34 @@ impl PersistentSparseCompactIntMatrixBuilder {
} }
pub fn close(self) -> io::Result<()> { pub fn close(self) -> io::Result<()> {
let Self { support, dir, singleton_values, multi_values, multi_row_offsets } = self; let Self {
support,
dir,
singleton_values,
multi_values,
multi_row_offsets,
} = self;
support.close()?; support.close()?;
let mut sv = PersistentCompactIntVecBuilder::new(singleton_values.len(), &singleton_values_path(&dir))?; let mut sv = PersistentCompactIntVecBuilder::new(
singleton_values.len(),
&singleton_values_path(&dir),
)?;
for (i, &v) in singleton_values.iter().enumerate() { for (i, &v) in singleton_values.iter().enumerate() {
sv.set(i, v); sv.set(i, v);
} }
sv.close()?; sv.close()?;
let mut mv = PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?; let mut mv =
PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?;
for (i, &v) in multi_values.iter().enumerate() { for (i, &v) in multi_values.iter().enumerate() {
mv.set(i, v); mv.set(i, v);
} }
mv.close()?; mv.close()?;
let universe = multi_values.len() as u64 + 1; let universe = multi_values.len() as u64 + 1;
let mut mo = EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?; let mut mo =
EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?;
for &o in &multi_row_offsets { for &o in &multi_row_offsets {
mo.push(o); mo.push(o);
} }
@@ -544,7 +604,7 @@ impl PersistentSparseCompactIntMatrixBuilder {
/// half-written one). Idempotent — a no-op if `singleton_values.pciv` /// half-written one). Idempotent — a no-op if `singleton_values.pciv`
/// already exists. /// already exists.
pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> { pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
use crate::intmatrix::{col_path, ColumnarCompactIntMatrix, PackedCompactIntMatrix}; use crate::intmatrix::{ColumnarCompactIntMatrix, PackedCompactIntMatrix, col_path};
if is_present(dir) { if is_present(dir) {
return Ok(()); return Ok(());
@@ -552,12 +612,12 @@ pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
let packed_path = dir.join("matrix.pcmx"); let packed_path = dir.join("matrix.pcmx");
if packed_path.exists() { if packed_path.exists() {
let dense = PersistentCompactIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?); let dense = PersistentIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?);
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?; PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
drop(dense); drop(dense);
fs::remove_file(&packed_path)?; fs::remove_file(&packed_path)?;
} else { } else {
let dense = PersistentCompactIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?); let dense = PersistentIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?);
let n_cols = dense.n_cols(); let n_cols = dense.n_cols();
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?; PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
drop(dense); drop(dense);
+30 -17
View File
@@ -1,25 +1,26 @@
use tempfile::tempdir; use tempfile::tempdir;
use crate::{ use crate::{
ColGroup, MatrixGroupOps, ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentIntMatrix, PersistentCompactIntMatrixBuilder,
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
}; };
use crate::{PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder}; use crate::{PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder};
// ── helpers ─────────────────────────────────────────────────────────────────── // ── helpers ───────────────────────────────────────────────────────────────────
fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) { fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
let n = cols.first().map_or(0, |c| c.len()); let n = cols.first().map_or(0, |c| c.len());
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap(); let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
for &col in cols { for &col in cols {
let mut cb = b.add_col().unwrap(); let mut cb = b.add_col().unwrap();
for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); } for (slot, &v) in col.iter().enumerate() {
cb.set(slot, v);
}
cb.close().unwrap(); cb.close().unwrap();
} }
b.close().unwrap(); b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); let m = PersistentIntMatrix::open(dir.path()).unwrap();
(dir, m) (dir, m)
} }
@@ -30,7 +31,9 @@ fn make_bit_matrix(cols: &[&[bool]]) -> (tempfile::TempDir, PersistentBitMatrix)
let mut b = PersistentBitMatrixBuilder::new(n, &presence).unwrap(); let mut b = PersistentBitMatrixBuilder::new(n, &presence).unwrap();
for &col in cols { for &col in cols {
let mut cb = b.add_col().unwrap(); let mut cb = b.add_col().unwrap();
for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); } for (slot, &v) in col.iter().enumerate() {
cb.set(slot, v);
}
cb.close().unwrap(); cb.close().unwrap();
} }
b.close().unwrap(); b.close().unwrap();
@@ -115,9 +118,13 @@ fn mask_with_zeros_selected_slots() {
// count vec [10, 20, 30, 40], mask [T, F, T, F] → [10, 0, 30, 0] // count vec [10, 20, 30, 40], mask [T, F, T, F] → [10, 0, 30, 0]
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap(); let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
v.set(0, 10); v.set(1, 20); v.set(2, 30); v.set(3, 40); v.set(0, 10);
v.set(1, 20);
v.set(2, 30);
v.set(3, 40);
let mut mask = PersistentBitVecBuilder::new(4, &dir.path().join("m.pbiv")).unwrap(); let mut mask = PersistentBitVecBuilder::new(4, &dir.path().join("m.pbiv")).unwrap();
mask.set(0, true); mask.set(2, true); mask.set(0, true);
mask.set(2, true);
v.mask_with(mask.view()); v.mask_with(mask.view());
v.close().unwrap(); v.close().unwrap();
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap(); let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
@@ -132,9 +139,12 @@ fn mask_with_overflow_slot_zeroed() {
// overflow slot (value 500) masked out → removed from overflow, primary=0 // overflow slot (value 500) masked out → removed from overflow, primary=0
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let mut v = PersistentCompactIntVecBuilder::new(3, &dir.path().join("v.pciv")).unwrap(); let mut v = PersistentCompactIntVecBuilder::new(3, &dir.path().join("v.pciv")).unwrap();
v.set(0, 10); v.set(1, 500); v.set(2, 5); v.set(0, 10);
v.set(1, 500);
v.set(2, 5);
let mut mask = PersistentBitVecBuilder::new(3, &dir.path().join("m.pbiv")).unwrap(); let mut mask = PersistentBitVecBuilder::new(3, &dir.path().join("m.pbiv")).unwrap();
mask.set(0, true); mask.set(2, true); // slot 1 masked out mask.set(0, true);
mask.set(2, true); // slot 1 masked out
v.mask_with(mask.view()); v.mask_with(mask.view());
v.close().unwrap(); v.close().unwrap();
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap(); let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
@@ -142,14 +152,20 @@ fn mask_with_overflow_slot_zeroed() {
assert_eq!(r.get(1), 0); assert_eq!(r.get(1), 0);
assert_eq!(r.get(2), 5); assert_eq!(r.get(2), 5);
let ov: Vec<_> = r.view().overflow_entries().collect(); let ov: Vec<_> = r.view().overflow_entries().collect();
assert!(ov.is_empty(), "overflow entry for masked-out slot should be gone"); assert!(
ov.is_empty(),
"overflow entry for masked-out slot should be gone"
);
} }
#[test] #[test]
fn mask_with_all_ones_is_noop() { fn mask_with_all_ones_is_noop() {
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap(); let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
v.set(0, 300); v.set(1, 1); v.set(2, 0); v.set(3, 42); v.set(0, 300);
v.set(1, 1);
v.set(2, 0);
v.set(3, 42);
let mask = PersistentBitVecBuilder::new_ones(4, &dir.path().join("m.pbiv")).unwrap(); let mask = PersistentBitVecBuilder::new_ones(4, &dir.path().join("m.pbiv")).unwrap();
v.mask_with(mask.view()); v.mask_with(mask.view());
v.close().unwrap(); v.close().unwrap();
@@ -184,10 +200,7 @@ fn bit_partial_group_presence_count() {
#[test] #[test]
fn bit_partial_group_any() { fn bit_partial_group_any() {
// col0=[T,F,F], col1=[F,F,T], group {0,1}: any = [T, F, T] // col0=[T,F,F], col1=[F,F,T], group {0,1}: any = [T, F, T]
let (_d, m) = make_bit_matrix(&[ let (_d, m) = make_bit_matrix(&[&[true, false, false], &[false, false, true]]);
&[true, false, false],
&[false, false, true],
]);
let g = ColGroup::new("g", vec![0, 1]); let g = ColGroup::new("g", vec![0, 1]);
let result = m.partial_group_any(&g, 1).unwrap(); let result = m.partial_group_any(&g, 1).unwrap();
assert_eq!(result.get(0), true); assert_eq!(result.get(0), true);
+102 -38
View File
@@ -1,9 +1,12 @@
use tempfile::tempdir; use tempfile::tempdir;
use crate::{pack_compact_int_matrix, pack_sparse_compact_int_matrix, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder, StorageKind};
use crate::traits::CountPartials; use crate::traits::CountPartials;
use crate::{
PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder,
PersistentIntMatrix, StorageKind, pack_compact_int_matrix, pack_sparse_compact_int_matrix,
};
fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) { fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
let n = cols.first().map_or(0, |c| c.len()); let n = cols.first().map_or(0, |c| c.len());
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap(); let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
@@ -15,7 +18,7 @@ fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatri
cb.close().unwrap(); cb.close().unwrap();
} }
b.close().unwrap(); b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); let m = PersistentIntMatrix::open(dir.path()).unwrap();
(dir, m) (dir, m)
} }
@@ -31,7 +34,7 @@ fn single_col_roundtrip() {
col.close().unwrap(); col.close().unwrap();
b.close().unwrap(); b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 1); assert_eq!(m.n_cols(), 1);
assert_eq!(m.n(), 4); assert_eq!(m.n(), 4);
assert_eq!(&*m.row(0), &[10u32]); assert_eq!(&*m.row(0), &[10u32]);
@@ -45,14 +48,18 @@ fn two_cols_roundtrip() {
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(3, &dir.path().join("counts")).unwrap(); let mut b = PersistentCompactIntMatrixBuilder::new(3, &dir.path().join("counts")).unwrap();
let mut col0 = b.add_col().unwrap(); let mut col0 = b.add_col().unwrap();
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3); col0.set(0, 1);
col0.set(1, 2);
col0.set(2, 3);
col0.close().unwrap(); col0.close().unwrap();
let mut col1 = b.add_col().unwrap(); let mut col1 = b.add_col().unwrap();
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30); col1.set(0, 10);
col1.set(1, 20);
col1.set(2, 30);
col1.close().unwrap(); col1.close().unwrap();
b.close().unwrap(); b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 2); assert_eq!(m.n_cols(), 2);
assert_eq!(&*m.row(0), &[1u32, 10]); assert_eq!(&*m.row(0), &[1u32, 10]);
assert_eq!(&*m.row(1), &[2u32, 20]); assert_eq!(&*m.row(1), &[2u32, 20]);
@@ -66,7 +73,9 @@ fn resume_continues_n_cols_and_appends_columns() {
let mut b = PersistentCompactIntMatrixBuilder::new(3, &counts).unwrap(); let mut b = PersistentCompactIntMatrixBuilder::new(3, &counts).unwrap();
let mut col0 = b.add_col().unwrap(); let mut col0 = b.add_col().unwrap();
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3); col0.set(0, 1);
col0.set(1, 2);
col0.set(2, 3);
col0.close().unwrap(); col0.close().unwrap();
b.close().unwrap(); b.close().unwrap();
@@ -74,11 +83,13 @@ fn resume_continues_n_cols_and_appends_columns() {
assert_eq!(resumed.n(), 3); assert_eq!(resumed.n(), 3);
assert_eq!(resumed.n_cols(), 1); assert_eq!(resumed.n_cols(), 1);
let mut col1 = resumed.add_col().unwrap(); let mut col1 = resumed.add_col().unwrap();
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30); col1.set(0, 10);
col1.set(1, 20);
col1.set(2, 30);
col1.close().unwrap(); col1.close().unwrap();
resumed.close().unwrap(); resumed.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 2); assert_eq!(m.n_cols(), 2);
assert_eq!(&*m.row(0), &[1u32, 10]); assert_eq!(&*m.row(0), &[1u32, 10]);
assert_eq!(&*m.row(1), &[2u32, 20]); assert_eq!(&*m.row(1), &[2u32, 20]);
@@ -90,17 +101,21 @@ fn resume_twice_keeps_appending() {
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let counts = dir.path().join("counts"); let counts = dir.path().join("counts");
PersistentCompactIntMatrixBuilder::new(2, &counts).unwrap().close().unwrap(); PersistentCompactIntMatrixBuilder::new(2, &counts)
.unwrap()
.close()
.unwrap();
for v in [1u32, 2, 3] { for v in [1u32, 2, 3] {
let mut b = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap(); let mut b = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap();
let mut col = b.add_col().unwrap(); let mut col = b.add_col().unwrap();
col.set(0, v); col.set(1, v * 10); col.set(0, v);
col.set(1, v * 10);
col.close().unwrap(); col.close().unwrap();
b.close().unwrap(); b.close().unwrap();
} }
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 3); assert_eq!(m.n_cols(), 3);
assert_eq!(&*m.row(0), &[1u32, 2, 3]); assert_eq!(&*m.row(0), &[1u32, 2, 3]);
assert_eq!(&*m.row(1), &[10u32, 20, 30]); assert_eq!(&*m.row(1), &[10u32, 20, 30]);
@@ -111,11 +126,12 @@ fn col_accessor() {
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(2, &dir.path().join("counts")).unwrap(); let mut b = PersistentCompactIntMatrixBuilder::new(2, &dir.path().join("counts")).unwrap();
let mut col0 = b.add_col().unwrap(); let mut col0 = b.add_col().unwrap();
col0.set(0, 5); col0.set(1, 7); col0.set(0, 5);
col0.set(1, 7);
col0.close().unwrap(); col0.close().unwrap();
b.close().unwrap(); b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.col(0).get(0), 5); assert_eq!(m.col(0).get(0), 5);
assert_eq!(m.col(0).get(1), 7); assert_eq!(m.col(0).get(1), 7);
} }
@@ -126,7 +142,7 @@ fn zero_cols_roundtrip() {
let b = PersistentCompactIntMatrixBuilder::new(10, &dir.path().join("counts")).unwrap(); let b = PersistentCompactIntMatrixBuilder::new(10, &dir.path().join("counts")).unwrap();
b.close().unwrap(); b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 0); assert_eq!(m.n_cols(), 0);
assert_eq!(m.n(), 10); assert_eq!(m.n(), 10);
} }
@@ -139,7 +155,9 @@ fn bray_dist_matrix_symmetry_and_diagonal() {
let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]); let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]);
let dm = m.bray_dist_matrix(); let dm = m.bray_dist_matrix();
let n = m.n_cols(); let n = m.n_cols();
for i in 0..n { assert_eq!(dm[[i, i]], 0.0, "diagonal"); } for i in 0..n {
assert_eq!(dm[[i, i]], 0.0, "diagonal");
}
for i in 0..n { for i in 0..n {
for j in 0..n { for j in 0..n {
assert!((dm[[i, j]] - dm[[j, i]]).abs() < 1e-12, "symmetry"); assert!((dm[[i, j]] - dm[[j, i]]).abs() < 1e-12, "symmetry");
@@ -189,7 +207,11 @@ fn partial_bray_dist_matrix_consistent() {
for i in 0..n { for i in 0..n {
for j in i + 1..n { for j in i + 1..n {
let denom = col_sums[i] + col_sums[j]; let denom = col_sums[i] + col_sums[j];
let dist = if denom == 0 { 0.0 } else { 1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64 }; let dist = if denom == 0 {
0.0
} else {
1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64
};
let expected = m.col(i).bray_dist(m.col(j)); let expected = m.col(i).bray_dist(m.col(j));
assert!((dist - expected).abs() < 1e-12, "[{i},{j}]"); assert!((dist - expected).abs() < 1e-12, "[{i},{j}]");
} }
@@ -254,8 +276,13 @@ fn partial_relfreq_bray_matches_full() {
// partial[i,j] = sum_min_relfreq; full[i,j] = 1 - sum_min_relfreq (off-diagonal only) // partial[i,j] = sum_min_relfreq; full[i,j] = 1 - sum_min_relfreq (off-diagonal only)
for i in 0..n { for i in 0..n {
for j in 0..n { for j in 0..n {
if i == j { continue; } if i == j {
assert!((partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12, "[{i},{j}]"); continue;
}
assert!(
(partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12,
"[{i},{j}]"
);
} }
} }
} }
@@ -270,7 +297,10 @@ fn partial_relfreq_euclidean_matches_full() {
for i in 0..n { for i in 0..n {
for j in 0..n { for j in 0..n {
// partial = squared euclidean; full = sqrt(partial) // partial = squared euclidean; full = sqrt(partial)
assert!((partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12, "[{i},{j}]"); assert!(
(partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12,
"[{i},{j}]"
);
} }
} }
} }
@@ -285,7 +315,10 @@ fn partial_hellinger_matches_full() {
for i in 0..n { for i in 0..n {
for j in 0..n { for j in 0..n {
// partial / sqrt(2) gives the Hellinger distance // partial / sqrt(2) gives the Hellinger distance
assert!((partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12, "[{i},{j}]"); assert!(
(partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12,
"[{i},{j}]"
);
} }
} }
} }
@@ -295,7 +328,7 @@ fn col_view_packed_values() {
// Build Columnar with overflow values (≥ 255), pack, reopen as Packed, exercise col_view(). // Build Columnar with overflow values (≥ 255), pack, reopen as Packed, exercise col_view().
let (dir, _col) = make_matrix(&[&[10, 300, 500], &[200, 50, 1000]]); let (dir, _col) = make_matrix(&[&[10, 300, 500], &[200, 50, 1000]]);
pack_compact_int_matrix(&dir.path().join("counts")).unwrap(); pack_compact_int_matrix(&dir.path().join("counts")).unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); let m = PersistentIntMatrix::open(dir.path()).unwrap();
// col 0: [10, 300, 500] — two overflow slots // col 0: [10, 300, 500] — two overflow slots
let v0 = m.col_view(0); let v0 = m.col_view(0);
@@ -326,7 +359,7 @@ fn col_view_packed_matches_columnar() {
// Re-build in a separate dir so we can pack without touching m_col's files. // Re-build in a separate dir so we can pack without touching m_col's files.
let (dir_pack, _) = make_matrix(data); let (dir_pack, _) = make_matrix(data);
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap(); pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap(); let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap();
for c in 0..data.len() { for c in 0..data.len() {
let col_ref = m_col.col(c); let col_ref = m_col.col(c);
@@ -355,7 +388,7 @@ fn nonzero_iter_matches_row() {
let (dir_col, m_col) = make_matrix(data); let (dir_col, m_col) = make_matrix(data);
let (dir_pack, _) = make_matrix(data); let (dir_pack, _) = make_matrix(data);
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap(); pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap(); let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap();
let slots = [4usize, 0, 3, 1]; let slots = [4usize, 0, 3, 1];
let mut expected: Vec<(usize, usize, u32)> = Vec::new(); let mut expected: Vec<(usize, usize, u32)> = Vec::new();
@@ -390,7 +423,7 @@ fn sparse_roundtrip_matches_columnar() {
let (dir_col, m_col) = make_matrix(data); let (dir_col, m_col) = make_matrix(data);
let (dir_sparse, _) = make_matrix(data); let (dir_sparse, _) = make_matrix(data);
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap(); pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap(); let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse); assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
assert_eq!(m_sparse.n(), m_col.n()); assert_eq!(m_sparse.n(), m_col.n());
@@ -441,7 +474,7 @@ fn sparse_count_partials_match_dense() {
let (dir_col, m_col) = make_matrix(data); let (dir_col, m_col) = make_matrix(data);
let (dir_sparse, _) = make_matrix(data); let (dir_sparse, _) = make_matrix(data);
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap(); pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap(); let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse); assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
let n = m_col.n_cols(); let n = m_col.n_cols();
@@ -461,7 +494,11 @@ fn sparse_count_partials_match_dense() {
let eucl_sparse = m_sparse.partial_euclidean_dist_matrix(); let eucl_sparse = m_sparse.partial_euclidean_dist_matrix();
for i in 0..n { for i in 0..n {
for j in 0..n { for j in 0..n {
close(eucl_col[[i, j]], eucl_sparse[[i, j]], &format!("partial_euclidean[{i},{j}]")); close(
eucl_col[[i, j]],
eucl_sparse[[i, j]],
&format!("partial_euclidean[{i},{j}]"),
);
} }
} }
@@ -469,9 +506,16 @@ fn sparse_count_partials_match_dense() {
// (support shortcut), and >1 (general row-major path). // (support shortcut), and >1 (general row-major path).
for threshold in [0u32, 1, 2, 3] { for threshold in [0u32, 1, 2, 3] {
let (inter_col, union_col) = m_col.partial_threshold_jaccard_dist_matrix(threshold); let (inter_col, union_col) = m_col.partial_threshold_jaccard_dist_matrix(threshold);
let (inter_sparse, union_sparse) = m_sparse.partial_threshold_jaccard_dist_matrix(threshold); let (inter_sparse, union_sparse) =
assert_eq!(inter_col, inter_sparse, "partial_threshold_jaccard inter, threshold={threshold}"); m_sparse.partial_threshold_jaccard_dist_matrix(threshold);
assert_eq!(union_col, union_sparse, "partial_threshold_jaccard union, threshold={threshold}"); assert_eq!(
inter_col, inter_sparse,
"partial_threshold_jaccard inter, threshold={threshold}"
);
assert_eq!(
union_col, union_sparse,
"partial_threshold_jaccard union, threshold={threshold}"
);
} }
// partial_relfreq_bray // partial_relfreq_bray
@@ -479,7 +523,11 @@ fn sparse_count_partials_match_dense() {
let rfb_sparse = m_sparse.partial_relfreq_bray_dist_matrix(&global); let rfb_sparse = m_sparse.partial_relfreq_bray_dist_matrix(&global);
for i in 0..n { for i in 0..n {
for j in 0..n { for j in 0..n {
close(rfb_col[[i, j]], rfb_sparse[[i, j]], &format!("partial_relfreq_bray[{i},{j}]")); close(
rfb_col[[i, j]],
rfb_sparse[[i, j]],
&format!("partial_relfreq_bray[{i},{j}]"),
);
} }
} }
@@ -488,7 +536,11 @@ fn sparse_count_partials_match_dense() {
let rfe_sparse = m_sparse.partial_relfreq_euclidean_dist_matrix(&global); let rfe_sparse = m_sparse.partial_relfreq_euclidean_dist_matrix(&global);
for i in 0..n { for i in 0..n {
for j in 0..n { for j in 0..n {
close(rfe_col[[i, j]], rfe_sparse[[i, j]], &format!("partial_relfreq_euclidean[{i},{j}]")); close(
rfe_col[[i, j]],
rfe_sparse[[i, j]],
&format!("partial_relfreq_euclidean[{i},{j}]"),
);
} }
} }
@@ -497,7 +549,11 @@ fn sparse_count_partials_match_dense() {
let hel_sparse = m_sparse.partial_hellinger_euclidean_dist_matrix(&global); let hel_sparse = m_sparse.partial_hellinger_euclidean_dist_matrix(&global);
for i in 0..n { for i in 0..n {
for j in 0..n { for j in 0..n {
close(hel_col[[i, j]], hel_sparse[[i, j]], &format!("partial_hellinger[{i},{j}]")); close(
hel_col[[i, j]],
hel_sparse[[i, j]],
&format!("partial_hellinger[{i},{j}]"),
);
} }
} }
@@ -514,7 +570,7 @@ fn sparse_roundtrip_from_packed() {
let (dir_sparse, _) = make_matrix(data); let (dir_sparse, _) = make_matrix(data);
pack_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap(); pack_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap(); pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap(); let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse); assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
for slot in 0..m_col.n() { for slot in 0..m_col.n() {
assert_eq!(&*m_sparse.row(slot), &*m_col.row(slot), "slot={slot}"); assert_eq!(&*m_sparse.row(slot), &*m_col.row(slot), "slot={slot}");
@@ -540,7 +596,10 @@ fn partial_relfreq_bray_additive_across_split() {
let n = m_full.n_cols(); let n = m_full.n_cols();
for i in 0..n { for i in 0..n {
for j in 0..n { for j in 0..n {
assert!((combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12, "[{i},{j}]"); assert!(
(combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12,
"[{i},{j}]"
);
} }
} }
} }
@@ -551,7 +610,9 @@ fn make_pciv(counts: &[u32]) -> (tempfile::TempDir, PersistentCompactIntVec) {
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let path = dir.path().join("c.pciv"); let path = dir.path().join("c.pciv");
let mut b = PersistentCompactIntVecBuilder::new(counts.len(), &path).unwrap(); let mut b = PersistentCompactIntVecBuilder::new(counts.len(), &path).unwrap();
for (i, &v) in counts.iter().enumerate() { b.set(i, v); } for (i, &v) in counts.iter().enumerate() {
b.set(i, v);
}
b.close().unwrap(); b.close().unwrap();
let r = PersistentCompactIntVec::open(&path).unwrap(); let r = PersistentCompactIntVec::open(&path).unwrap();
(dir, r) (dir, r)
@@ -592,7 +653,10 @@ fn pciv_collect_slots_values_empty() {
fn pciv_collect_slots_values_out_of_bounds_panics() { fn pciv_collect_slots_values_out_of_bounds_panics() {
let (_dir, v) = make_pciv(&[10u32, 20]); let (_dir, v) = make_pciv(&[10u32, 20]);
let result = std::panic::catch_unwind(|| v.collect_slots_values(&[0, 2])); let result = std::panic::catch_unwind(|| v.collect_slots_values(&[0, 2]));
assert!(result.is_err(), "collect_slots_values should panic on out-of-bounds slot"); assert!(
result.is_err(),
"collect_slots_values should panic on out-of-bounds slot"
);
} }
// IntSliceView collect_slots_values (same logic, exercised through the view) // IntSliceView collect_slots_values (same logic, exercised through the view)
+47 -47
View File
@@ -1,13 +1,14 @@
use tempfile::tempdir; use tempfile::tempdir;
use crate::{ use crate::{
pack_compact_int_matrix, pack_sparse_compact_int_matrix, ColumnWeights, PersistentCompactIntMatrix, ColumnWeights, PersistentIntMatrix, PersistentCompactIntMatrixBuilder,
PersistentCompactIntMatrixBuilder, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
pack_compact_int_matrix, pack_sparse_compact_int_matrix,
}; };
/// Builds a dense `PersistentCompactIntMatrix` from column-major `u32` data /// Builds a dense `PersistentCompactIntMatrix` from column-major `u32` data
/// — mirrors `tests/intmatrix.rs`'s own `make_matrix` helper. /// — mirrors `tests/intmatrix.rs`'s own `make_matrix` helper.
fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) { fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
let n = cols.first().map_or(0, |c| c.len()); let n = cols.first().map_or(0, |c| c.len());
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let counts_dir = dir.path().join("counts"); let counts_dir = dir.path().join("counts");
@@ -20,17 +21,21 @@ fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix
cb.close().unwrap(); cb.close().unwrap();
} }
b.close().unwrap(); b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); let m = PersistentIntMatrix::open(dir.path()).unwrap();
(dir, m) (dir, m)
} }
/// Builds a sparse int matrix directly from row-major `u32` data (one /// Builds a sparse int matrix directly from row-major `u32` data (one
/// slice per row, `n_cols` values each) — mirrors `tests/sparse.rs`'s /// slice per row, `n_cols` values each) — mirrors `tests/sparse.rs`'s
/// `make_sparse` helper. /// `make_sparse` helper.
fn make_sparse(rows: &[&[u32]], n_cols: usize) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) { fn make_sparse(
rows: &[&[u32]],
n_cols: usize,
) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) {
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let sparse_dir = dir.path().join("sparse"); let sparse_dir = dir.path().join("sparse");
let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap(); let mut b =
PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap();
let mut cols = Vec::new(); let mut cols = Vec::new();
let mut values = Vec::new(); let mut values = Vec::new();
for row in rows { for row in rows {
@@ -70,11 +75,7 @@ fn basic_roundtrip_singletons_and_multi() {
#[test] #[test]
fn get_matches_row() { fn get_matches_row() {
let rows: Vec<&[u32]> = vec![ let rows: Vec<&[u32]> = vec![&[3, 0, 5], &[0, 8, 0], &[1, 2, 4]];
&[3, 0, 5],
&[0, 8, 0],
&[1, 2, 4],
];
let (_dir, m) = make_sparse(&rows, 3); let (_dir, m) = make_sparse(&rows, 3);
for (slot, row) in rows.iter().enumerate() { for (slot, row) in rows.iter().enumerate() {
for (c, &expected) in row.iter().enumerate() { for (c, &expected) in row.iter().enumerate() {
@@ -85,11 +86,7 @@ fn get_matches_row() {
#[test] #[test]
fn fill_row_matches_row() { fn fill_row_matches_row() {
let rows: Vec<&[u32]> = vec![ let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]];
&[9, 0, 2],
&[0, 4, 0],
&[1, 1, 1],
];
let (_dir, m) = make_sparse(&rows, 3); let (_dir, m) = make_sparse(&rows, 3);
let mut buf = vec![0u32; 3]; let mut buf = vec![0u32; 3];
for slot in 0..3 { for slot in 0..3 {
@@ -120,11 +117,7 @@ fn fill_sub_matrix_matches_row() {
#[test] #[test]
fn nonzero_iter_matches_row() { fn nonzero_iter_matches_row() {
let rows: Vec<&[u32]> = vec![ let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]];
&[9, 0, 2],
&[0, 4, 0],
&[1, 1, 1],
];
let (_dir, m) = make_sparse(&rows, 3); let (_dir, m) = make_sparse(&rows, 3);
let slots = [2usize, 0, 1]; let slots = [2usize, 0, 1];
let mut expected: Vec<(usize, usize, u32)> = Vec::new(); let mut expected: Vec<(usize, usize, u32)> = Vec::new();
@@ -143,12 +136,7 @@ fn nonzero_iter_matches_row() {
#[test] #[test]
fn sum_and_count_nonzero_match_naive() { fn sum_and_count_nonzero_match_naive() {
let rows: Vec<&[u32]> = vec![ let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1], &[0, 0, 6]];
&[9, 0, 2],
&[0, 4, 0],
&[1, 1, 1],
&[0, 0, 6],
];
let (_dir, m) = make_sparse(&rows, 3); let (_dir, m) = make_sparse(&rows, 3);
let mut expected_sum = [0u64; 3]; let mut expected_sum = [0u64; 3];
let mut expected_count = [0u64; 3]; let mut expected_count = [0u64; 3];
@@ -193,14 +181,21 @@ fn reopen_after_close_matches_original() {
let rows: Vec<Vec<u32>> = (0..500) let rows: Vec<Vec<u32>> = (0..500)
.map(|i| { .map(|i| {
(0..37) (0..37)
.map(|c| if (i * 7 + c * 3) % 11 == 0 { ((i + c) % 250 + 1) as u32 } else { 0 }) .map(|c| {
if (i * 7 + c * 3) % 11 == 0 {
((i + c) % 250 + 1) as u32
} else {
0
}
})
.collect() .collect()
}) })
.collect(); .collect();
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
let sparse_dir = dir.path().join("sparse"); let sparse_dir = dir.path().join("sparse");
{ {
let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap(); let mut b =
PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap();
let mut cols = Vec::new(); let mut cols = Vec::new();
let mut values = Vec::new(); let mut values = Vec::new();
for row in &rows { for row in &rows {
@@ -229,10 +224,7 @@ fn reopen_after_close_matches_original() {
fn overflow_values_roundtrip() { fn overflow_values_roundtrip() {
// Values >= 255 exercise the PersistentCompactIntVec overflow path in // Values >= 255 exercise the PersistentCompactIntVec overflow path in
// both the singleton and multi value streams. // both the singleton and multi value streams.
let rows: Vec<&[u32]> = vec![ let rows: Vec<&[u32]> = vec![&[1000, 0, 0], &[0, 50_000, 300]];
&[1000, 0, 0],
&[0, 50_000, 300],
];
let (_dir, m) = make_sparse(&rows, 3); let (_dir, m) = make_sparse(&rows, 3);
for (slot, &expected) in rows.iter().enumerate() { for (slot, &expected) in rows.iter().enumerate() {
assert_eq!(&*m.row(slot), expected, "row {slot}"); assert_eq!(&*m.row(slot), expected, "row {slot}");
@@ -246,14 +238,13 @@ fn overflow_values_roundtrip() {
/// `tests::sparse::pack_sparse_bit_matrix_transposes_columnar_directly`. /// `tests::sparse::pack_sparse_bit_matrix_transposes_columnar_directly`.
#[test] #[test]
fn pack_sparse_compact_int_matrix_transposes_columnar_directly() { fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
let cols: &[&[u32]] = &[ let cols: &[&[u32]] = &[&[9, 0, 2, 0], &[0, 4, 0, 7], &[1, 1, 300, 1]];
&[9, 0, 2, 0],
&[0, 4, 0, 7],
&[1, 1, 300, 1],
];
let (dir, dense) = make_dense(cols); let (dir, dense) = make_dense(cols);
let counts_dir = dir.path().join("counts"); let counts_dir = dir.path().join("counts");
assert!(!counts_dir.join("matrix.pcmx").exists(), "still Columnar before packing"); assert!(
!counts_dir.join("matrix.pcmx").exists(),
"still Columnar before packing"
);
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect(); let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
let n_cols = dense.n_cols(); let n_cols = dense.n_cols();
@@ -262,9 +253,18 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
pack_sparse_compact_int_matrix(&counts_dir).unwrap(); pack_sparse_compact_int_matrix(&counts_dir).unwrap();
assert!(counts_dir.join("singleton_values.pciv").exists()); assert!(counts_dir.join("singleton_values.pciv").exists());
assert!(!counts_dir.join("meta.json").exists(), "columnar meta.json cleaned up"); assert!(
assert!(!counts_dir.join("col_000000.pciv").exists(), "columnar column files cleaned up"); !counts_dir.join("meta.json").exists(),
assert!(!counts_dir.join("matrix.pcmx").exists(), "never materialised"); "columnar meta.json cleaned up"
);
assert!(
!counts_dir.join("col_000000.pciv").exists(),
"columnar column files cleaned up"
);
assert!(
!counts_dir.join("matrix.pcmx").exists(),
"never materialised"
);
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap(); let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
assert_eq!(sparse.n(), expected_rows.len()); assert_eq!(sparse.n(), expected_rows.len());
@@ -282,10 +282,7 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
/// it's still used as the transpose source and cleaned up afterward. /// it's still used as the transpose source and cleaned up afterward.
#[test] #[test]
fn pack_sparse_compact_int_matrix_from_already_packed_matrix() { fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
let cols: &[&[u32]] = &[ let cols: &[&[u32]] = &[&[9, 0, 2], &[0, 4, 300]];
&[9, 0, 2],
&[0, 4, 300],
];
let (dir, dense) = make_dense(cols); let (dir, dense) = make_dense(cols);
let counts_dir = dir.path().join("counts"); let counts_dir = dir.path().join("counts");
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect(); let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
@@ -296,7 +293,10 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
assert!(counts_dir.join("matrix.pcmx").exists()); assert!(counts_dir.join("matrix.pcmx").exists());
pack_sparse_compact_int_matrix(&counts_dir).unwrap(); pack_sparse_compact_int_matrix(&counts_dir).unwrap();
assert!(!counts_dir.join("matrix.pcmx").exists(), "packed intermediate cleaned up"); assert!(
!counts_dir.join("matrix.pcmx").exists(),
"packed intermediate cleaned up"
);
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap(); let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
assert_eq!(sparse.n_cols(), n_cols); assert_eq!(sparse.n_cols(), n_cols);
+7 -4
View File
@@ -26,7 +26,7 @@ use std::path::{Path, PathBuf};
use crate::layer::utils::LAYERNAME_SUFFIX; use crate::layer::utils::LAYERNAME_SUFFIX;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
use obikseq::CanonicalKmer; use obikseq::CanonicalKmer;
use crate::index::error::OKIResult; use crate::index::error::OKIResult;
@@ -63,7 +63,7 @@ pub enum KmerLayer {
Count { Count {
dir: PathBuf, dir: PathBuf,
id: usize, id: usize,
layer: TypedLayer<PersistentCompactIntMatrix>, layer: TypedLayer<PersistentIntMatrix>,
}, },
Presence { Presence {
dir: PathBuf, dir: PathBuf,
@@ -99,7 +99,7 @@ impl KmerLayer {
already_open => return Ok(already_open), already_open => return Ok(already_open),
}; };
if dir.join(COUNTS_DIR).exists() { if dir.join(COUNTS_DIR).exists() {
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(&dir)?; let layer = TypedLayer::<PersistentIntMatrix>::open(&dir)?;
Ok(KmerLayer::Count { dir, id, layer }) Ok(KmerLayer::Count { dir, id, layer })
} else { } else {
let layer = TypedLayer::<PersistentBitMatrix>::open(&dir)?; let layer = TypedLayer::<PersistentBitMatrix>::open(&dir)?;
@@ -221,7 +221,10 @@ impl KmerLayer {
/// format-agnostic column-major fetch, delegating to whichever matrix /// format-agnostic column-major fetch, delegating to whichever matrix
/// this layer actually holds (see `obicompactvec::PersistentBitMatrix`/ /// this layer actually holds (see `obicompactvec::PersistentBitMatrix`/
/// `PersistentCompactIntMatrix::nonzero_iter`). /// `PersistentCompactIntMatrix::nonzero_iter`).
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> { pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
match self { match self {
KmerLayer::Presence { layer, .. } => layer.nonzero_iter(slots), KmerLayer::Presence { layer, .. } => layer.nonzero_iter(slots),
KmerLayer::Count { layer, .. } => layer.nonzero_iter(slots), KmerLayer::Count { layer, .. } => layer.nonzero_iter(slots),
+63 -22
View File
@@ -1,6 +1,6 @@
use super::*; use super::*;
use obicompactvec::PersistentSparseBitMatrixBuilder; use obicompactvec::PersistentSparseBitMatrixBuilder;
use obikseq::{set_k, Unitig}; use obikseq::{Unitig, set_k};
use obiskio::DEFAULT_BLOCK_BITS; use obiskio::DEFAULT_BLOCK_BITS;
use tempfile::tempdir; use tempfile::tempdir;
@@ -13,7 +13,8 @@ fn write_unitigs(dir: &Path, seqs: &[&[u8]]) {
} }
fn all_canonical_kmers(dir: &Path) -> Vec<CanonicalKmer> { fn all_canonical_kmers(dir: &Path) -> Vec<CanonicalKmer> {
UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE)).unwrap() UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE))
.unwrap()
.iter_indexed_canonical_kmers() .iter_indexed_canonical_kmers()
.map(|(kmer, _, _)| kmer) .map(|(kmer, _, _)| kmer)
.collect() .collect()
@@ -34,13 +35,17 @@ fn canonical_kmer_iter_matches_reader() {
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAAACGT", b"TTTTGCA"]); write_unitigs(dir.path(), &[b"AAAACGT", b"TTTTGCA"]);
let from_iter: Vec<CanonicalKmer> = obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE)) let from_iter: Vec<CanonicalKmer> =
obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE))
.unwrap() .unwrap()
.collect(); .collect();
let from_reader: Vec<CanonicalKmer> = all_canonical_kmers(dir.path()); let from_reader: Vec<CanonicalKmer> = all_canonical_kmers(dir.path());
assert_eq!(from_iter.len(), from_reader.len(), "different kmer counts"); assert_eq!(from_iter.len(), from_reader.len(), "different kmer counts");
assert_eq!(from_iter, from_reader, "CanonicalKmerIter and UnitigFileReader disagree"); assert_eq!(
from_iter, from_reader,
"CanonicalKmerIter and UnitigFileReader disagree"
);
} }
// ── Generic `TypedLayer<D>` over dense vs. sparse presence storage ─────────────── // ── Generic `TypedLayer<D>` over dense vs. sparse presence storage ───────────────
@@ -64,9 +69,14 @@ fn presence_layer_generic_over_sparse_matches_dense() {
// kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be // kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be
// biologically meaningful, just the same on both sides of the // biologically meaningful, just the same on both sides of the
// dense/sparse comparison below. // dense/sparse comparison below.
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| { TypedLayer::<PersistentBitMatrix>::build_presence(
(kmer.raw().wrapping_add(g as u64)) % 2 == 0 dir.path(),
}).unwrap(); DEFAULT_BLOCK_BITS,
&mode,
n_genomes,
|kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0,
)
.unwrap();
let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap(); let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
assert!(dense_layer.n_cols() >= 1); assert!(dense_layer.n_cols() >= 1);
@@ -76,7 +86,10 @@ fn presence_layer_generic_over_sparse_matches_dense() {
// same directory, distinct filenames — see // same directory, distinct filenames — see
// `DevDocMD/architecture/siblings.md`'s sparse-matrix section). // `DevDocMD/architecture/siblings.md`'s sparse-matrix section).
let dense_matrix = obicompactvec::PersistentBitMatrix::open(dir.path()).unwrap(); let dense_matrix = obicompactvec::PersistentBitMatrix::open(dir.path()).unwrap();
PersistentSparseBitMatrixBuilder::build_from_dense(&dense_matrix, &dir.path().join(PRESENCE_DIR)) PersistentSparseBitMatrixBuilder::build_from_dense(
&dense_matrix,
&dir.path().join(PRESENCE_DIR),
)
.unwrap() .unwrap()
.close() .close()
.unwrap(); .unwrap();
@@ -88,7 +101,10 @@ fn presence_layer_generic_over_sparse_matches_dense() {
let n = dense_layer.n(); let n = dense_layer.n();
assert_eq!(n, sparse_layer.n()); assert_eq!(n, sparse_layer.n());
let slots: Vec<usize> = (0..n).collect(); let slots: Vec<usize> = (0..n).collect();
assert_eq!(dense_layer.sub_matrix(&slots), sparse_layer.sub_matrix(&slots)); assert_eq!(
dense_layer.sub_matrix(&slots),
sparse_layer.sub_matrix(&slots)
);
// The matrix-agnostic, MPHF-only surface (from the generic // The matrix-agnostic, MPHF-only surface (from the generic
// `impl<D: LayerData> TypedLayer<D>`) must also agree: same kmer set, // `impl<D: LayerData> TypedLayer<D>`) must also agree: same kmer set,
@@ -124,15 +140,25 @@ fn count_layer_transparently_reads_sparse_after_pack() {
// Deterministic, arbitrary count function — same shape as the presence // Deterministic, arbitrary count function — same shape as the presence
// test's arbitrary predicate, just producing a small count instead of a // test's arbitrary predicate, just producing a small count instead of a
// bool. // bool.
TypedLayer::<()>::build_with_matrix(dir.path(), DEFAULT_BLOCK_BITS, &mode, false, n_genomes, |kmer| { TypedLayer::<()>::build_with_matrix(
dir.path(),
DEFAULT_BLOCK_BITS,
&mode,
false,
n_genomes,
|kmer| {
(0..n_genomes) (0..n_genomes)
.map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32) .map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32)
.collect() .collect()
}) },
)
.unwrap(); .unwrap();
let dense_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap(); let dense_layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
assert_eq!(dense_layer.storage_kind(), obicompactvec::StorageKind::Columnar); assert_eq!(
dense_layer.storage_kind(),
obicompactvec::StorageKind::Columnar
);
let n = dense_layer.n(); let n = dense_layer.n();
let slots: Vec<usize> = (0..n).collect(); let slots: Vec<usize> = (0..n).collect();
let dense_sub = dense_layer.sub_matrix(&slots); let dense_sub = dense_layer.sub_matrix(&slots);
@@ -146,8 +172,11 @@ fn count_layer_transparently_reads_sparse_after_pack() {
// in production. // in production.
obicompactvec::pack_sparse_compact_int_matrix(&dir.path().join(COUNTS_DIR)).unwrap(); obicompactvec::pack_sparse_compact_int_matrix(&dir.path().join(COUNTS_DIR)).unwrap();
let sparse_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap(); let sparse_layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
assert_eq!(sparse_layer.storage_kind(), obicompactvec::StorageKind::Sparse); assert_eq!(
sparse_layer.storage_kind(),
obicompactvec::StorageKind::Sparse
);
assert_eq!(sparse_layer.n(), n); assert_eq!(sparse_layer.n(), n);
assert_eq!(sparse_layer.n_cols(), dense_layer.n_cols()); assert_eq!(sparse_layer.n_cols(), dense_layer.n_cols());
assert_eq!(sparse_layer.sub_matrix(&slots), dense_sub); assert_eq!(sparse_layer.sub_matrix(&slots), dense_sub);
@@ -167,13 +196,20 @@ fn count_layer_reports_count_content_and_columnar_storage() {
set_k(4); set_k(4);
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAAACGT"]); write_unitigs(dir.path(), &[b"AAAACGT"]);
TypedLayer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1) TypedLayer::<PersistentIntMatrix>::build(dir.pat
.unwrap(); h(), DEFAUL
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap(); T_BLOCK_BITS, &Index
Mode::Exact, |_| 1)
,
).unwrap();
let layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
assert_eq!(layer.content(), LayerContent::Count); assert_eq!(layer.content(), LayerContent::Count);
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar); assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
assert_eq!(layer.evidence_kind(), crate::layer::mphf_layer::EvidenceKind::Exact); assert_eq!(
layer.evidence_kind(),
crate::layer::mphf_layer::EvidenceKind::Exact
);
} }
#[test] #[test]
@@ -181,9 +217,14 @@ fn presence_layer_reports_presence_content_and_columnar_storage() {
set_k(4); set_k(4);
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]); write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| { TypedLayer::<PersistentBitMatrix>::build_presence(
(kmer.raw().wrapping_add(g as u64)) % 2 == 0 dir.path(),
}).unwrap(); DEFAULT_BLOCK_BITS,
&IndexMode::Exact,
2,
|kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0,
)
.unwrap();
let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap(); let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
assert_eq!(layer.content(), LayerContent::Presence); assert_eq!(layer.content(), LayerContent::Presence);
+17 -11
View File
@@ -1,7 +1,7 @@
// use crate::layer::utils::layer_dir; // use crate::layer::utils::layer_dir;
use obicompactvec::{ use obicompactvec::{
BinaryMatrix, ColBuilder, MatrixBuilder, PersistentBitMatrix, PersistentBitMatrixBuilder, BinaryMatrix, ColBuilder, MatrixBuilder, PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentSparseBitMatrix, PersistentCompactIntMatrixBuilder, PersistentIntMatrix, PersistentSparseBitMatrix,
}; };
use obikseq::CanonicalKmer; use obikseq::CanonicalKmer;
use obiskio::{UnitigFileReader, UnitigFileWriter}; use obiskio::{UnitigFileReader, UnitigFileWriter};
@@ -47,10 +47,10 @@ impl LayerData for () {
fn read(&self, _slot: usize) {} fn read(&self, _slot: usize) {}
} }
impl LayerData for PersistentCompactIntMatrix { impl LayerData for PersistentIntMatrix {
type Item = Box<[u32]>; type Item = Box<[u32]>;
fn open(layer_dir: &Path) -> OKIResult<Self> { fn open(layer_dir: &Path) -> OKIResult<Self> {
PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io) PersistentIntMatrix::open(layer_dir).map_err(OKIError::Io)
} }
fn read(&self, slot: usize) -> Box<[u32]> { fn read(&self, slot: usize) -> Box<[u32]> {
self.row(slot) self.row(slot)
@@ -112,7 +112,7 @@ pub trait HasLayerContent {
const CONTENT: LayerContent; const CONTENT: LayerContent;
} }
impl HasLayerContent for PersistentCompactIntMatrix { impl HasLayerContent for PersistentIntMatrix {
const CONTENT: LayerContent = LayerContent::Count; const CONTENT: LayerContent = LayerContent::Count;
} }
@@ -135,9 +135,9 @@ pub trait HasStorageKind {
fn storage_kind(&self) -> obicompactvec::StorageKind; fn storage_kind(&self) -> obicompactvec::StorageKind;
} }
impl HasStorageKind for PersistentCompactIntMatrix { impl HasStorageKind for PersistentIntMatrix {
fn storage_kind(&self) -> obicompactvec::StorageKind { fn storage_kind(&self) -> obicompactvec::StorageKind {
PersistentCompactIntMatrix::storage_kind(self) PersistentIntMatrix::storage_kind(self)
} }
} }
@@ -342,7 +342,7 @@ impl TypedLayer<()> {
// ── Mode 2 — count matrix ───────────────────────────────────────────────────── // ── Mode 2 — count matrix ─────────────────────────────────────────────────────
impl TypedLayer<PersistentCompactIntMatrix> { impl TypedLayer<PersistentIntMatrix> {
pub fn build( pub fn build(
out_dir: &Path, out_dir: &Path,
block_bits: u8, block_bits: u8,
@@ -377,12 +377,12 @@ impl TypedLayer<PersistentCompactIntMatrix> {
// ── Mode 2 — count matrix column append ────────────────────────────────────── // ── Mode 2 — count matrix column append ──────────────────────────────────────
impl TypedLayer<PersistentCompactIntMatrix> { impl TypedLayer<PersistentIntMatrix> {
pub fn append_genome_column( pub fn append_genome_column(
layer_dir: &Path, layer_dir: &Path,
value_of: impl Fn(usize) -> u32, value_of: impl Fn(usize) -> u32,
) -> OKIResult<()> { ) -> OKIResult<()> {
PersistentCompactIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of) PersistentIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of)
.map_err(OKIError::Io) .map_err(OKIError::Io)
} }
@@ -418,7 +418,10 @@ impl TypedLayer<PersistentCompactIntMatrix> {
/// Every nonzero `(idx into slots, col, value)` triple among `slots` — /// Every nonzero `(idx into slots, col, value)` triple among `slots` —
/// delegates to `PersistentCompactIntMatrix::nonzero_iter`. /// delegates to `PersistentCompactIntMatrix::nonzero_iter`.
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> { pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
self.data.nonzero_iter(slots) self.data.nonzero_iter(slots)
} }
} }
@@ -477,7 +480,10 @@ impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix + obicompactvec::ColumnWeig
impl TypedLayer<PersistentBitMatrix> { impl TypedLayer<PersistentBitMatrix> {
/// Every nonzero `(idx into slots, col, value)` triple among `slots` — /// Every nonzero `(idx into slots, col, value)` triple among `slots` —
/// delegates to `PersistentBitMatrix::nonzero_iter`. /// delegates to `PersistentBitMatrix::nonzero_iter`.
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> { pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
self.data.nonzero_iter(slots) self.data.nonzero_iter(slots)
} }
@@ -24,14 +24,14 @@ use std::fs;
use std::io; use std::io;
use std::path::Path; use std::path::Path;
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use cacheline_ef::{CachelineEf, CachelineEfVec}; use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::prelude::*; use epserde::prelude::*;
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec}; use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn; use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode; use obikindex::layer::IndexMode;
use obikindex::layer::{KmerLayer, TypedLayer}; use obikindex::layer::{KmerLayer, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult}; use obikindex::{KmerIndex, OKIError, OKIResult};
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use obiskio::{SKError, SKFileMeta, SKFileReader}; use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64}; use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
@@ -139,7 +139,9 @@ impl PrivateBuilder for KmerIndex {
mode: &IndexMode, mode: &IndexMode,
block_bits: u8, block_bits: u8,
) -> Result<usize, SKError> { ) -> Result<usize, SKError> {
let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?; let layer0 = self
.layer0(i)
.map_err(|e| io::Error::other(e.to_string()))?;
let layer0_dir = layer0.dir(); let layer0_dir = layer0.dir();
let dedup_path = layer0.dereplicated_superkmers_path(); let dedup_path = layer0.dereplicated_superkmers_path();
if !dedup_path.exists() { if !dedup_path.exists() {
@@ -193,15 +195,12 @@ impl PrivateBuilder for KmerIndex {
let n_kmers = if with_counts { let n_kmers = if with_counts {
let n = write_graph_as_unitigs(g, layer0_dir) let n = write_graph_as_unitigs(g, layer0_dir)
.map_err(|e| io::Error::other(e.to_string()))?; .map_err(|e| io::Error::other(e.to_string()))?;
TypedLayer::<PersistentCompactIntMatrix>::build( TypedLayer::<PersistentIntMatrix>::build(layer0_dir, block_bits, mode, |kmer| {
layer0_dir, match (&mphf1_opt, &counts1_opt) {
block_bits,
mode,
|kmer| match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())), (Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1, _ => 1,
}, }
) })
.map_err(|e| io::Error::other(e.to_string()))?; .map_err(|e| io::Error::other(e.to_string()))?;
n n
} else { } else {
+15 -7
View File
@@ -34,7 +34,7 @@ fn presence(mode: MergeMode) -> bool {
mod matrix_builder_tests { mod matrix_builder_tests {
use tempfile::tempdir; use tempfile::tempdir;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
use super::{ColBuilder, MatrixBuilder}; use super::{ColBuilder, MatrixBuilder};
@@ -90,7 +90,7 @@ mod matrix_builder_tests {
col.close().unwrap(); col.close().unwrap();
mb.close().unwrap(); mb.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 2); assert_eq!(m.n_cols(), 2);
assert_eq!(&*m.row(0), &[0u32, 7]); assert_eq!(&*m.row(0), &[0u32, 7]);
assert_eq!(&*m.row(1), &[0u32, 42]); assert_eq!(&*m.row(1), &[0u32, 42]);
@@ -422,8 +422,13 @@ pub(crate) fn merge_partition(
move |data: Pass2Data, move |data: Pass2Data,
push: &PipelineSender<Result<Pass2Data, PipelineError>>, push: &PipelineSender<Result<Pass2Data, PipelineError>>,
delta: &PipelineSender<isize>| { delta: &PipelineSender<isize>| {
if let Pass2Data::SrcLayer((col_offset, src_n, unitigs_path, src_layer, _guard)) = if let Pass2Data::SrcLayer((
data col_offset,
src_n,
unitigs_path,
src_layer,
_guard,
)) = data
{ {
// _guard dropped at end of block, releasing the slot. // _guard dropped at end of block, releasing the slot.
// `src_layer` (MPHF + matrix) was already opened up // `src_layer` (MPHF + matrix) was already opened up
@@ -455,7 +460,9 @@ pub(crate) fn merge_partition(
} }
} }
if !batch.is_empty() { if !batch.is_empty() {
push.send(Ok(Pass2Data::RawBatch((col_offset, src_n, src_layer, batch)))) push.send(Ok(Pass2Data::RawBatch((
col_offset, src_n, src_layer, batch,
))))
.ok(); .ok();
count += 1; count += 1;
} }
@@ -480,8 +487,9 @@ pub(crate) fn merge_partition(
// per kmer — matches the underlying matrix's own // per kmer — matches the underlying matrix's own
// column-major layout. // column-major layout.
let slots = src_layer.hash_batch(&kmers); let slots = src_layer.hash_batch(&kmers);
let mut cols: Vec<Vec<u32>> = let mut cols: Vec<Vec<u32>> = (0..src_n)
(0..src_n).map(|_| Vec::with_capacity(kmers.len())).collect(); .map(|_| Vec::with_capacity(kmers.len()))
.collect();
src_layer.fill_sub_matrix(&slots, &mut cols); src_layer.fill_sub_matrix(&slots, &mut cols);
// Membership against dst, grouped by dst layer // Membership against dst, grouped by dst layer
+4 -4
View File
@@ -1,15 +1,15 @@
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
use obikidxcache::LayeredStore;
use obikindex::OKIResult; use obikindex::OKIResult;
use obikindex::layer::open_data; use obikindex::layer::open_data;
use obikidxcache::LayeredStore;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikindex::load_meta;
use obikindex::KmerIndex; use obikindex::KmerIndex;
use obikindex::load_meta;
impl KmerIndex { impl KmerIndex {
/// Open all count matrices for partition `part`, one per layer. /// Open all count matrices for partition `part`, one per layer.
/// Layers without a `counts/` directory are skipped. /// Layers without a `counts/` directory are skipped.
pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentCompactIntMatrix>> { pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentIntMatrix>> {
let index_dir = self.index_dir(part); let index_dir = self.index_dir(part);
if !index_dir.exists() { if !index_dir.exists() {
return Ok(LayeredStore::new(vec![])); return Ok(LayeredStore::new(vec![]));
+18 -7
View File
@@ -11,8 +11,8 @@ use std::io;
use std::path::Path; use std::path::Path;
use obicompactvec::{ use obicompactvec::{
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix, ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix, PersistentIntMatrix, TempBitVec,
PersistentCompactIntMatrix, TempBitVec, TempCompactIntVec, TempCompactIntVec,
}; };
use obikindex::layer::{KmerLayer, LayerContent}; use obikindex::layer::{KmerLayer, LayerContent};
use obikindex::{KmerIndex, OKIError, OKIResult}; use obikindex::{KmerIndex, OKIError, OKIResult};
@@ -45,7 +45,9 @@ impl AggOp {
"sum" => Ok(AggOp::Sum), "sum" => Ok(AggOp::Sum),
"min" => Ok(AggOp::Min), "min" => Ok(AggOp::Min),
"max" => Ok(AggOp::Max), "max" => Ok(AggOp::Max),
other => Err(format!("unknown aggregation operator: {other}; valid: any, all, none, sum, min, max")), other => Err(format!(
"unknown aggregation operator: {other}; valid: any, all, none, sum, min, max"
)),
} }
} }
@@ -75,7 +77,11 @@ enum AggResult {
Int(TempCompactIntVec), Int(TempCompactIntVec),
} }
fn compute_group(mat: &dyn MatrixGroupOps, spec: &OutputCol, threshold: u32) -> io::Result<AggResult> { fn compute_group(
mat: &dyn MatrixGroupOps,
spec: &OutputCol,
threshold: u32,
) -> io::Result<AggResult> {
let g = ColGroup::new(spec.label.clone(), spec.indices.clone()); let g = ColGroup::new(spec.label.clone(), spec.indices.clone());
Ok(match spec.op { Ok(match spec.op {
AggOp::Any => AggResult::Bit(mat.partial_group_any(&g, threshold)?), AggOp::Any => AggResult::Bit(mat.partial_group_any(&g, threshold)?),
@@ -145,18 +151,23 @@ pub(crate) fn select_partition(
let group_mat: Box<dyn MatrixGroupOps> = match src_layer.content() { let group_mat: Box<dyn MatrixGroupOps> = match src_layer.content() {
LayerContent::Count => { LayerContent::Count => {
Box::new(PersistentCompactIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?) Box::new(PersistentIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
} }
LayerContent::Presence => { LayerContent::Presence => {
Box::new(PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?) Box::new(PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
} }
}; };
let data_subdir = if output_presence { "presence" } else { "counts" }; let data_subdir = if output_presence {
"presence"
} else {
"counts"
};
let data_dir = dst_layer_dir.join(data_subdir); let data_dir = dst_layer_dir.join(data_subdir);
fs::create_dir_all(&data_dir).map_err(OKIError::Io)?; fs::create_dir_all(&data_dir).map_err(OKIError::Io)?;
let mut builder = MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?; let mut builder =
MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?;
for spec in specs { for spec in specs {
let r = compute_group(group_mat.as_ref(), spec, threshold).map_err(OKIError::Io)?; let r = compute_group(group_mat.as_ref(), spec, threshold).map_err(OKIError::Io)?;
add_result(&mut builder, r).map_err(OKIError::Io)?; add_result(&mut builder, r).map_err(OKIError::Io)?;