OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Push zunrplorkwkt #70

Merged
coissac merged 93 commits from push-zunrplorkwkt into main 2026-08-28 23:15:38 +00:00
Conversation 0 Commits 93 Files Changed 428
+779 -369
14 changed files with 779 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 4b6005962e - Show all commits
src/obicompactvec/examples/compare_sparse_dense_count.rs
+29 -10
View File
@@ -13,20 +13,30 @@ use std::error::Error;
use std::path::Path;
use std::time::Instant;
use obicompactvec::{PersistentCompactIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder};
use obicompactvec::{
PersistentIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
};
fn main() -> Result<(), Box<dyn Error>> {
let layer_dir = std::env::args().nth(1).expect("usage: compare_sparse_dense_count <layer_dir>");
let layer_dir = std::env::args()
.nth(1)
.expect("usage: compare_sparse_dense_count <layer_dir>");
let layer_dir = Path::new(&layer_dir);
let t0 = Instant::now();
let dense = PersistentCompactIntMatrix::open(layer_dir)?;
println!("dense ouverte en {:?} ({} lignes x {} colonnes)", t0.elapsed(), dense.n(), dense.n_cols());
let dense = PersistentIntMatrix::open(layer_dir)?;
println!(
"dense ouverte en {:?} ({} lignes x {} colonnes)",
t0.elapsed(),
dense.n(),
dense.n_cols()
);
let out_dir = tempfile::tempdir()?;
let t0 = Instant::now();
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?.finish()?;
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?
.finish()?;
println!("build_from_dense: {:?}", t0.elapsed());
compare(&dense, &sparse)?;
@@ -36,7 +46,10 @@ fn main() -> Result<(), Box<dyn Error>> {
Ok(())
}
fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactIntMatrix) -> Result<(), Box<dyn Error>> {
fn compare(
dense: &PersistentIntMatrix,
sparse: &PersistentSparseCompactIntMatrix,
) -> Result<(), Box<dyn Error>> {
let n = dense.n();
let n_cols = dense.n_cols();
assert_eq!(n, sparse.n(), "n mismatch");
@@ -63,7 +76,10 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI
}
}
}
println!("comparaison case-a-case: {:?} ({total_cells} cellules)", t0.elapsed());
println!(
"comparaison case-a-case: {:?} ({total_cells} cellules)",
t0.elapsed()
);
if let Some((slot, col, d, s)) = first_mismatch {
eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}");
@@ -78,7 +94,7 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI
/// for what's actually stored) — sparsity, singleton-row fraction, and the
/// value-magnitude buckets this crate's overflow encoding is tuned around
/// (< 127, < 255, >= 255).
fn content_stats(dense: &PersistentCompactIntMatrix) {
fn content_stats(dense: &PersistentIntMatrix) {
let n = dense.n();
let n_cols = dense.n_cols();
let total_cells = n as u64 * n_cols as u64;
@@ -128,7 +144,10 @@ fn content_stats(dense: &PersistentCompactIntMatrix) {
100.0 * singleton_rows as f64 / n as f64
);
if nonzero_cells > 0 {
println!("valeur moyenne (non-nulles): {:.2}", sum as f64 / nonzero_cells as f64);
println!(
"valeur moyenne (non-nulles): {:.2}",
sum as f64 / nonzero_cells as f64
);
println!("valeur max: {max_value}");
println!(
"valeurs < 127: {under_127} ({:.3}% des non-nulles)",
@@ -162,7 +181,7 @@ fn dir_size(dir: &Path) -> std::io::Result<u64> {
fn compaction_stats(
dense_layer_dir: &Path,
sparse_dir: &Path,
dense: &PersistentCompactIntMatrix,
dense: &PersistentIntMatrix,
_sparse: &PersistentSparseCompactIntMatrix,
) -> Result<(), Box<dyn Error>> {
let dense_size = dir_size(&dense_layer_dir.join("counts"))?;
src/obicompactvec/src/intmatrix.rs
+288 -111
View File
@@ -5,17 +5,19 @@ use std::path::{Path, PathBuf};
use memmap2::Mmap;
use ndarray::{Array1, Array2};
use crate::bitmatrix::{fill_row_generic, par_col_reduce, pairwise2_matrix, pairwise_matrix, row_generic};
use crate::bitmatrix::{
fill_row_generic, pairwise_matrix, pairwise2_matrix, par_col_reduce, row_generic,
};
use crate::builder::PersistentCompactIntVecBuilder;
use crate::colgroup::{chunked_presence_count, ColGroup, MatrixGroupOps};
use crate::colgroup::{ColGroup, MatrixGroupOps, chunked_presence_count};
use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE};
use crate::meta::MatrixMeta;
use crate::reader::PersistentCompactIntVec;
use crate::sparse_intmatrix::{is_present as sparse_is_present, PersistentSparseCompactIntMatrix};
use crate::sparse_intmatrix::{PersistentSparseCompactIntMatrix, is_present as sparse_is_present};
use crate::storage_kind::StorageKind;
use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
use crate::views::{nonzero_triples, IntSliceView};
use crate::views::{IntSliceView, nonzero_triples};
pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf {
dir.join(format!("col_{col:06}.pciv"))
@@ -25,7 +27,7 @@ pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf {
pub struct ColumnarCompactIntMatrix {
cols: Vec<PersistentCompactIntVec>,
n: usize,
n: usize,
}
impl ColumnarCompactIntMatrix {
@@ -38,11 +40,17 @@ impl ColumnarCompactIntMatrix {
}
#[inline]
pub(crate) fn n(&self) -> usize { self.n }
pub(crate) fn n(&self) -> usize {
self.n
}
#[inline]
pub(crate) fn n_cols(&self) -> usize { self.cols.len() }
pub(crate) fn n_cols(&self) -> usize {
self.cols.len()
}
#[inline]
pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec { &self.cols[c] }
pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec {
&self.cols[c]
}
#[inline]
pub(crate) fn row(&self, slot: usize) -> Box<[u32]> {
@@ -63,34 +71,64 @@ impl ColumnarCompactIntMatrix {
}
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_bray_dist(self.col(j)))
pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_bray_dist(self.col(j))
})
}
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_euclidean_dist(self.col(j)))
pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_euclidean_dist(self.col(j))
})
}
pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
pairwise2_matrix(self.n_cols(), |i, j| self.col(i).partial_threshold_jaccard_dist(self.col(j), threshold))
pub(crate) fn partial_threshold_jaccard_dist_matrix(
&self,
threshold: u32,
) -> (Array2<u64>, Array2<u64>) {
pairwise2_matrix(self.n_cols(), |i, j| {
self.col(i)
.partial_threshold_jaccard_dist(self.col(j), threshold)
})
}
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_relfreq_bray_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
self.col(i).partial_relfreq_bray_dist(
self.col(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pub(crate) fn partial_relfreq_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_relfreq_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
self.col(i).partial_relfreq_euclidean_dist(
self.col(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pub(crate) fn partial_hellinger_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_hellinger_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
self.col(i).partial_hellinger_euclidean_dist(
self.col(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
pub(crate) fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> {
let mut meta = MatrixMeta::load(dir)?;
let mut b = PersistentCompactIntVecBuilder::new(meta.n, &col_path(dir, meta.n_cols))?;
for slot in 0..meta.n { b.set(slot, value_of(slot)); }
for slot in 0..meta.n {
b.set(slot, value_of(slot));
}
b.close()?;
meta.n_cols += 1;
meta.save(dir)
@@ -99,19 +137,19 @@ impl ColumnarCompactIntMatrix {
// ── PackedCompactIntMatrix ────────────────────────────────────────────────────
const PCMX_MAGIC: [u8; 4] = *b"PCMX";
const PCMX_HEADER: usize = 24; // magic(4) + pad(4) + n_rows(8) + n_cols(8)
const PCMX_MAGIC: [u8; 4] = *b"PCMX";
const PCMX_HEADER: usize = 24; // magic(4) + pad(4) + n_rows(8) + n_cols(8)
struct ColInfo {
primary_start: usize,
data_offset: usize,
n_overflow: usize,
data_offset: usize,
n_overflow: usize,
}
pub struct PackedCompactIntMatrix {
mmap: Mmap,
n_rows: usize,
n_cols: usize,
mmap: Mmap,
n_rows: usize,
n_cols: usize,
columns: Vec<ColInfo>,
}
@@ -119,43 +157,65 @@ impl PackedCompactIntMatrix {
pub(crate) fn open(path: &Path) -> io::Result<Self> {
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
if mmap.len() < PCMX_HEADER {
return Err(io::Error::new(io::ErrorKind::InvalidData, "PCMX file too short"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"PCMX file too short",
));
}
if &mmap[0..4] != &PCMX_MAGIC {
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PCMX magic"));
}
let n_rows = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
let n_rows = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
let n_cols = u64::from_le_bytes(mmap[16..24].try_into().unwrap()) as usize;
let mut columns = Vec::with_capacity(n_cols);
for c in 0..n_cols {
let off_pos = PCMX_HEADER + c * 8;
let col_base = u64::from_le_bytes(mmap[off_pos..off_pos+8].try_into().unwrap()) as usize;
let n_ov = u64::from_le_bytes(mmap[col_base+16..col_base+24].try_into().unwrap()) as usize;
let n_pciv = u64::from_le_bytes(mmap[col_base+8..col_base+16].try_into().unwrap()) as usize;
let off_pos = PCMX_HEADER + c * 8;
let col_base =
u64::from_le_bytes(mmap[off_pos..off_pos + 8].try_into().unwrap()) as usize;
let n_ov =
u64::from_le_bytes(mmap[col_base + 16..col_base + 24].try_into().unwrap()) as usize;
let n_pciv =
u64::from_le_bytes(mmap[col_base + 8..col_base + 16].try_into().unwrap()) as usize;
let primary_start = col_base + HEADER_SIZE;
let data_offset = primary_start + n_pciv;
columns.push(ColInfo { primary_start, data_offset, n_overflow: n_ov });
let data_offset = primary_start + n_pciv;
columns.push(ColInfo {
primary_start,
data_offset,
n_overflow: n_ov,
});
}
Ok(Self { mmap, n_rows, n_cols, columns })
Ok(Self {
mmap,
n_rows,
n_cols,
columns,
})
}
#[inline]
pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> {
let ci = &self.columns[c];
let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
let overflow_raw = &self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE];
let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
let overflow_raw =
&self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE];
IntSliceView::new(primary, overflow_raw, ci.n_overflow, self.n_rows)
}
pub(crate) fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentCompactIntVecBuilder> {
pub(crate) fn col_persist(
&self,
c: usize,
path: &Path,
) -> io::Result<PersistentCompactIntVecBuilder> {
let view = self.col_view(c);
let overflow: std::collections::HashMap<usize, u32> = view.overflow_entries().collect();
PersistentCompactIntVecBuilder::from_raw_primary(view.primary_bytes(), overflow, path)
}
#[inline]
pub(crate) fn get(&self, col: usize, slot: usize) -> u32 { self.col_view(col).get(slot) }
pub(crate) fn get(&self, col: usize, slot: usize) -> u32 {
self.col_view(col).get(slot)
}
#[inline]
pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
@@ -180,27 +240,55 @@ impl PackedCompactIntMatrix {
// via `PersistentCompactIntVec`. No formula is re-derived here.
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_bray_dist(self.col_view(j)))
pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_bray_dist(self.col_view(j))
})
}
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_euclidean_dist(self.col_view(j)))
pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_euclidean_dist(self.col_view(j))
})
}
pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, t: u32) -> (Array2<u64>, Array2<u64>) {
pairwise2_matrix(self.n_cols, |i, j| self.col_view(i).partial_threshold_jaccard_dist(self.col_view(j), t))
pub(crate) fn partial_threshold_jaccard_dist_matrix(
&self,
t: u32,
) -> (Array2<u64>, Array2<u64>) {
pairwise2_matrix(self.n_cols, |i, j| {
self.col_view(i)
.partial_threshold_jaccard_dist(self.col_view(j), t)
})
}
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_relfreq_bray_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
self.col_view(i).partial_relfreq_bray_dist(
self.col_view(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pub(crate) fn partial_relfreq_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_relfreq_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
self.col_view(i).partial_relfreq_euclidean_dist(
self.col_view(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pub(crate) fn partial_hellinger_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_hellinger_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
self.col_view(i).partial_hellinger_euclidean_dist(
self.col_view(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
}
@@ -238,7 +326,9 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
// the columnar files are newer and must be (re-)packed, overwriting the
// stale one — never silently discarded as "leftover cleanup".
if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
for c in 0..meta.n_cols {
let _ = fs::remove_file(col_path(dir, c));
}
let _ = fs::remove_file(dir.join("meta.json"));
return Ok(());
}
@@ -250,32 +340,41 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
let header_size = (PCMX_HEADER + n_cols * 8) as u64;
let mut col_offset = header_size;
let mut offsets = Vec::with_capacity(n_cols);
for &size in &col_sizes { offsets.push(col_offset); col_offset += size; }
for &size in &col_sizes {
offsets.push(col_offset);
col_offset += size;
}
let tmp_path = dir.join("matrix.pcmx.tmp");
let mut out = BufWriter::new(File::create(&tmp_path)?);
out.write_all(&PCMX_MAGIC)?;
out.write_all(&[0u8; 4])?;
out.write_all(&(meta.n as u64).to_le_bytes())?;
out.write_all(&(n_cols as u64).to_le_bytes())?;
for &off in &offsets { out.write_all(&off.to_le_bytes())?; }
for c in 0..n_cols { io::copy(&mut File::open(col_path(dir, c))?, &mut out)?; }
for &off in &offsets {
out.write_all(&off.to_le_bytes())?;
}
for c in 0..n_cols {
io::copy(&mut File::open(col_path(dir, c))?, &mut out)?;
}
out.flush()?;
drop(out);
fs::rename(&tmp_path, &packed_path)?;
for c in 0..n_cols { fs::remove_file(col_path(dir, c))?; }
for c in 0..n_cols {
fs::remove_file(col_path(dir, c))?;
}
fs::remove_file(dir.join("meta.json"))?;
Ok(())
}
// ── PersistentCompactIntMatrix — public enum ──────────────────────────────────
pub enum PersistentCompactIntMatrix {
pub enum PersistentIntMatrix {
Columnar(ColumnarCompactIntMatrix),
Packed(PackedCompactIntMatrix),
Sparse(PersistentSparseCompactIntMatrix),
}
impl PersistentCompactIntMatrix {
impl PersistentIntMatrix {
/// Checks (in order): `counts/matrix.pcmx` → Packed; `counts/meta.json`
/// → Columnar; `counts/singleton_values.pciv` → Sparse. Mirrors
/// `PersistentBitMatrix::open`'s own Packed → Columnar → Sparse
@@ -283,17 +382,24 @@ impl PersistentCompactIntMatrix {
pub fn open(layer_dir: &Path) -> io::Result<Self> {
let counts_dir = layer_dir.join("counts");
if counts_dir.join("matrix.pcmx").exists() {
return Ok(Self::Packed(PackedCompactIntMatrix::open(&counts_dir.join("matrix.pcmx"))?));
return Ok(Self::Packed(PackedCompactIntMatrix::open(
&counts_dir.join("matrix.pcmx"),
)?));
}
if MatrixMeta::load(&counts_dir).is_ok() {
return Ok(Self::Columnar(ColumnarCompactIntMatrix::open(&counts_dir)?));
}
if sparse_is_present(&counts_dir) {
return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open(&counts_dir)?));
return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open(
&counts_dir,
)?));
}
Err(io::Error::new(
io::ErrorKind::NotFound,
format!("no count matrix found in {} — run 'obikmer upgrade'", layer_dir.display()),
format!(
"no count matrix found in {} — run 'obikmer upgrade'",
layer_dir.display()
),
))
}
@@ -303,8 +409,8 @@ impl PersistentCompactIntMatrix {
pub fn storage_kind(&self) -> StorageKind {
match self {
Self::Columnar(_) => StorageKind::Columnar,
Self::Packed(_) => StorageKind::Packed,
Self::Sparse(_) => StorageKind::Sparse,
Self::Packed(_) => StorageKind::Packed,
Self::Sparse(_) => StorageKind::Sparse,
}
}
@@ -330,11 +436,19 @@ impl PersistentCompactIntMatrix {
#[inline]
pub fn n(&self) -> usize {
match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows, Self::Sparse(m) => m.n() }
match self {
Self::Columnar(m) => m.n(),
Self::Packed(m) => m.n_rows,
Self::Sparse(m) => m.n(),
}
}
#[inline]
pub fn n_cols(&self) -> usize {
match self { Self::Columnar(m) => m.n_cols(), Self::Packed(m) => m.n_cols, Self::Sparse(m) => m.n_cols() }
match self {
Self::Columnar(m) => m.n_cols(),
Self::Packed(m) => m.n_cols,
Self::Sparse(m) => m.n_cols(),
}
}
#[inline]
@@ -349,30 +463,38 @@ impl PersistentCompactIntMatrix {
pub fn col_view(&self, c: usize) -> IntSliceView<'_> {
match self {
Self::Columnar(m) => m.col(c).view(),
Self::Packed(m) => m.col_view(c),
Self::Sparse(_) => panic!("col_view() not available on Sparse PersistentCompactIntMatrix"),
Self::Packed(m) => m.col_view(c),
Self::Sparse(_) => {
panic!("col_view() not available on Sparse PersistentCompactIntMatrix")
}
}
}
pub fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentCompactIntVecBuilder> {
match self {
Self::Columnar(m) => PersistentCompactIntVecBuilder::build_from(m.col(c), path),
Self::Packed(m) => m.col_persist(c, path),
Self::Sparse(_) => Err(io::Error::new(io::ErrorKind::Unsupported,
"col_persist not available on Sparse PersistentCompactIntMatrix")),
Self::Packed(m) => m.col_persist(c, path),
Self::Sparse(_) => Err(io::Error::new(
io::ErrorKind::Unsupported,
"col_persist not available on Sparse PersistentCompactIntMatrix",
)),
}
}
#[inline]
pub fn row(&self, slot: usize) -> Box<[u32]> {
match self { Self::Columnar(m) => m.row(slot), Self::Packed(m) => m.row(slot), Self::Sparse(m) => m.row(slot) }
match self {
Self::Columnar(m) => m.row(slot),
Self::Packed(m) => m.row(slot),
Self::Sparse(m) => m.row(slot),
}
}
#[inline]
pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
match self {
Self::Columnar(m) => m.fill_row(slot, buf),
Self::Packed(m) => m.fill_row(slot, buf),
Self::Sparse(m) => m.fill_row(slot, buf),
Self::Packed(m) => m.fill_row(slot, buf),
Self::Sparse(m) => m.fill_row(slot, buf),
}
}
@@ -418,73 +540,86 @@ impl PersistentCompactIntMatrix {
/// `Columnar`/`Packed` reuse the shared sorted-slot batching in
/// `views::nonzero_triples`. Boxed, not `impl Iterator`, because the
/// match arms are genuinely different concrete types.
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
match self {
Self::Sparse(m) => Box::new(m.nonzero_iter(slots)),
Self::Columnar(_) | Self::Packed(_) => {
Box::new(nonzero_triples(slots, self.n_cols(), |c| self.col_view(c), false))
}
Self::Columnar(_) | Self::Packed(_) => Box::new(nonzero_triples(
slots,
self.n_cols(),
|c| self.col_view(c),
false,
)),
}
}
#[inline]
pub fn sum(&self) -> Array1<u64> {
match self { Self::Columnar(m) => m.sum(), Self::Packed(m) => m.sum(), Self::Sparse(m) => m.sum() }
match self {
Self::Columnar(m) => m.sum(),
Self::Packed(m) => m.sum(),
Self::Sparse(m) => m.sum(),
}
}
#[inline]
pub fn count_nonzero(&self) -> Array1<u64> {
match self {
Self::Columnar(m) => m.count_nonzero(),
Self::Packed(m) => m.count_nonzero(),
Self::Sparse(m) => m.count_nonzero(),
Self::Packed(m) => m.count_nonzero(),
Self::Sparse(m) => m.count_nonzero(),
}
}
#[inline]
pub fn partial_bray_dist_matrix(&self) -> Array2<u64> {
match self {
Self::Columnar(m) => m.partial_bray_dist_matrix(),
Self::Packed(m) => m.partial_bray_dist_matrix(),
Self::Sparse(m) => CountPartials::partial_bray(m),
Self::Packed(m) => m.partial_bray_dist_matrix(),
Self::Sparse(m) => CountPartials::partial_bray(m),
}
}
#[inline]
pub fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
match self {
Self::Columnar(m) => m.partial_euclidean_dist_matrix(),
Self::Packed(m) => m.partial_euclidean_dist_matrix(),
Self::Sparse(m) => CountPartials::partial_euclidean(m),
Self::Packed(m) => m.partial_euclidean_dist_matrix(),
Self::Sparse(m) => CountPartials::partial_euclidean(m),
}
}
#[inline]
pub fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
pub fn partial_threshold_jaccard_dist_matrix(
&self,
threshold: u32,
) -> (Array2<u64>, Array2<u64>) {
match self {
Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
Self::Sparse(m) => CountPartials::partial_threshold_jaccard(m, threshold),
Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
Self::Sparse(m) => CountPartials::partial_threshold_jaccard(m, threshold),
}
}
#[inline]
pub fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
match self {
Self::Columnar(m) => m.partial_relfreq_bray_dist_matrix(col_sums),
Self::Packed(m) => m.partial_relfreq_bray_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_relfreq_bray(m, col_sums),
Self::Packed(m) => m.partial_relfreq_bray_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_relfreq_bray(m, col_sums),
}
}
#[inline]
pub fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
match self {
Self::Columnar(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums),
Self::Packed(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_relfreq_euclidean(m, col_sums),
Self::Packed(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_relfreq_euclidean(m, col_sums),
}
}
#[inline]
pub fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
match self {
Self::Columnar(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums),
Self::Packed(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_hellinger(m, col_sums),
Self::Packed(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_hellinger(m, col_sums),
}
}
#[inline]
@@ -497,40 +632,60 @@ impl PersistentCompactIntMatrix {
use crate::traits::{ColumnWeights, CountPartials};
impl ColumnWeights for PersistentCompactIntMatrix {
impl ColumnWeights for PersistentIntMatrix {
#[inline]
fn col_weights(&self) -> Array1<u64> { self.sum() }
fn col_weights(&self) -> Array1<u64> {
self.sum()
}
#[inline]
fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() }
fn partial_kmer_counts(&self) -> Array1<u64> {
self.count_nonzero()
}
}
impl CountPartials for PersistentCompactIntMatrix {
impl CountPartials for PersistentIntMatrix {
#[inline]
fn partial_bray(&self) -> Array2<u64> { self.partial_bray_dist_matrix() }
fn partial_bray(&self) -> Array2<u64> {
self.partial_bray_dist_matrix()
}
#[inline]
fn partial_euclidean(&self) -> Array2<f64> { self.partial_euclidean_dist_matrix() }
fn partial_euclidean(&self) -> Array2<f64> {
self.partial_euclidean_dist_matrix()
}
#[inline]
fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) { self.partial_threshold_jaccard_dist_matrix(t) }
fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) {
self.partial_threshold_jaccard_dist_matrix(t)
}
#[inline]
fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_bray_dist_matrix(g) }
fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> {
self.partial_relfreq_bray_dist_matrix(g)
}
#[inline]
fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_euclidean_dist_matrix(g) }
fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> {
self.partial_relfreq_euclidean_dist_matrix(g)
}
#[inline]
fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_hellinger_euclidean_dist_matrix(g) }
fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> {
self.partial_hellinger_euclidean_dist_matrix(g)
}
}
// ── Builder ───────────────────────────────────────────────────────────────────
pub struct PersistentCompactIntMatrixBuilder {
dir: PathBuf,
n: usize,
dir: PathBuf,
n: usize,
n_cols: usize,
}
impl PersistentCompactIntMatrixBuilder {
pub fn new(n: usize, dir: &Path) -> io::Result<Self> {
fs::create_dir_all(dir)?;
Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
Ok(Self {
dir: dir.to_path_buf(),
n,
n_cols: 0,
})
}
/// Resume appending columns to a Columnar matrix directory already
@@ -541,13 +696,21 @@ impl PersistentCompactIntMatrixBuilder {
/// on-disk column naming or `MatrixMeta` themselves.
pub fn resume(dir: &Path) -> io::Result<Self> {
let meta = MatrixMeta::load(dir)?;
Ok(Self { dir: dir.to_path_buf(), n: meta.n, n_cols: meta.n_cols })
Ok(Self {
dir: dir.to_path_buf(),
n: meta.n,
n_cols: meta.n_cols,
})
}
#[inline]
pub fn n(&self) -> usize { self.n }
pub fn n(&self) -> usize {
self.n
}
#[inline]
pub fn n_cols(&self) -> usize { self.n_cols }
pub fn n_cols(&self) -> usize {
self.n_cols
}
pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> {
let path = col_path(&self.dir, self.n_cols);
self.n_cols += 1;
@@ -569,14 +732,22 @@ impl PersistentCompactIntMatrixBuilder {
}
pub fn close(self) -> io::Result<()> {
MatrixMeta { n: self.n, n_cols: self.n_cols }.save(&self.dir)
MatrixMeta {
n: self.n,
n_cols: self.n_cols,
}
.save(&self.dir)
}
}
// ── MatrixGroupOps ────────────────────────────────────────────────────────────
impl MatrixGroupOps for PersistentCompactIntMatrix {
fn partial_group_presence_count(&self, g: &ColGroup, threshold: u32) -> io::Result<TempCompactIntVec> {
impl MatrixGroupOps for PersistentIntMatrix {
fn partial_group_presence_count(
&self,
g: &ColGroup,
threshold: u32,
) -> io::Result<TempCompactIntVec> {
chunked_presence_count(self.n(), &g.indices, |b, c| {
b.inc_predicate_fast(self.col_view(c), |v| v >= threshold)
})
@@ -585,7 +756,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
let n = self.n();
let mut result = TempCompactIntVecBuilder::new(n)?;
for &c in &g.indices { result.add(self.col_view(c)); }
for &c in &g.indices {
result.add(self.col_view(c));
}
result.freeze()
}
@@ -603,7 +776,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
let mut result = TempCompactIntVecBuilder::new(n)?;
if let Some((&first, rest)) = g.indices.split_first() {
result.add(self.col_view(first));
for &c in rest { result.min(self.col_view(c)); }
for &c in rest {
result.min(self.col_view(c));
}
}
result.freeze()
}
@@ -611,7 +786,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
let n = self.n();
let mut result = TempCompactIntVecBuilder::new(n)?;
for &c in &g.indices { result.max(self.col_view(c)); }
for &c in &g.indices {
result.max(self.col_view(c));
}
result.freeze()
}
}
src/obicompactvec/src/lib.rs
+20 -11
View File
@@ -1,41 +1,50 @@
mod bitvec;
mod bitmatrix;
mod bitvec;
mod builder;
mod colgroup;
mod eliasfano;
mod fixedintvec;
mod format;
mod rankselect;
mod intmatrix;
mod layer_meta;
mod matrix_builder;
mod meta;
mod mmap_file;
mod rankselect;
mod reader;
mod sparse_intmatrix;
mod storage_kind;
mod tempbitvec;
mod tempintvec;
mod views;
pub mod traits;
mod views;
pub use bitmatrix::{
PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix,
PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix,
};
pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
pub use eliasfano::{EliasFano, EliasFanoBuilder};
pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix};
pub use builder::PersistentCompactIntVecBuilder;
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
pub use eliasfano::{EliasFano, EliasFanoBuilder};
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
pub use intmatrix::{
PersistentCompactIntMatrixBuilder, PersistentIntMatrix, pack_compact_int_matrix,
};
pub use layer_meta::LayerMeta;
pub use matrix_builder::{ColBuilder, MatrixBuilder};
pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter};
pub use sparse_intmatrix::{PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder, pack_sparse_compact_int_matrix};
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
pub use reader::{Iter as CompactIntVecIter, PersistentCompactIntVec};
pub use sparse_intmatrix::{
PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
pack_sparse_compact_int_matrix,
};
pub use storage_kind::StorageKind;
pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials};
pub use views::{BitSliceView, BitSliceIter, IntSliceView, IntSliceViewIter};
pub use views::{BitSliceIter, BitSliceView, IntSliceView, IntSliceViewIter};
#[cfg(test)]
#[path = "tests/mod.rs"]
src/obicompactvec/src/sparse_intmatrix.rs
+101 -41
View File
@@ -35,11 +35,17 @@ use crate::builder::PersistentCompactIntVecBuilder;
use crate::eliasfano::{EliasFano, EliasFanoBuilder};
use crate::reader::PersistentCompactIntVec;
use crate::traits::{BitPartials, ColumnWeights, CountPartials};
use crate::{PersistentCompactIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
use crate::{PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
fn singleton_values_path(dir: &Path) -> PathBuf { dir.join("singleton_values.pciv") }
fn multi_values_path(dir: &Path) -> PathBuf { dir.join("multi_values.pciv") }
fn multi_offsets_base(dir: &Path) -> PathBuf { dir.join("multi_offsets") }
fn singleton_values_path(dir: &Path) -> PathBuf {
dir.join("singleton_values.pciv")
}
fn multi_values_path(dir: &Path) -> PathBuf {
dir.join("multi_values.pciv")
}
fn multi_offsets_base(dir: &Path) -> PathBuf {
dir.join("multi_offsets")
}
/// Lightweight disk probe: true iff a sparse count matrix has already been
/// written at `dir` — the marker [`PersistentCompactIntMatrix::open`]/
@@ -73,9 +79,13 @@ impl PersistentSparseCompactIntMatrix {
}
#[inline]
pub fn n(&self) -> usize { self.support.n() }
pub fn n(&self) -> usize {
self.support.n()
}
#[inline]
pub fn n_cols(&self) -> usize { self.support.n_cols() }
pub fn n_cols(&self) -> usize {
self.support.n_cols()
}
/// Calls `f(col, value)` once per non-zero column at `slot`, in
/// ascending column order — the shared decode branch behind every
@@ -104,7 +114,11 @@ impl PersistentSparseCompactIntMatrix {
/// absent).
pub fn get(&self, c: usize, slot: usize) -> u32 {
let mut found = 0u32;
self.for_each_cell_in_row(slot, |g, v| if g == c { found = v; });
self.for_each_cell_in_row(slot, |g, v| {
if g == c {
found = v;
}
});
found
}
@@ -138,27 +152,32 @@ impl PersistentSparseCompactIntMatrix {
/// Yields every `(idx into slots, col, value)` triple, row by row, in
/// `slots` order — mirrors
/// [`PersistentSparseBitMatrix::nonzero_iter`].
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator<Item = (usize, usize, u32)> + 'a {
pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> impl Iterator<Item = (usize, usize, u32)> + 'a {
let mut next_slot = 0usize;
let mut cur_idx = 0usize;
let mut buf: Vec<(usize, u32)> = Vec::new();
let mut buf_pos = 0usize;
std::iter::from_fn(move || loop {
if buf_pos < buf.len() {
let (g, v) = buf[buf_pos];
buf_pos += 1;
return Some((cur_idx, g, v));
std::iter::from_fn(move || {
loop {
if buf_pos < buf.len() {
let (g, v) = buf[buf_pos];
buf_pos += 1;
return Some((cur_idx, g, v));
}
if next_slot >= slots.len() {
return None;
}
cur_idx = next_slot;
let slot = slots[next_slot];
next_slot += 1;
buf.clear();
self.for_each_cell_in_row(slot, |g, v| buf.push((g, v)));
buf_pos = 0;
}
if next_slot >= slots.len() {
return None;
}
cur_idx = next_slot;
let slot = slots[next_slot];
next_slot += 1;
buf.clear();
self.for_each_cell_in_row(slot, |g, v| buf.push((g, v)));
buf_pos = 0;
})
}
@@ -203,7 +222,11 @@ impl PersistentSparseCompactIntMatrix {
fn count_geq(&self, threshold: u32) -> Array1<u64> {
let mut counts = vec![0u64; self.n_cols()];
for slot in 0..self.n() {
self.for_each_cell_in_row(slot, |g, v| if v >= threshold { counts[g] += 1; });
self.for_each_cell_in_row(slot, |g, v| {
if v >= threshold {
counts[g] += 1;
}
});
}
Array1::from(counts)
}
@@ -261,9 +284,13 @@ impl PersistentSparseCompactIntMatrix {
impl ColumnWeights for PersistentSparseCompactIntMatrix {
#[inline]
fn col_weights(&self) -> Array1<u64> { self.sum() }
fn col_weights(&self) -> Array1<u64> {
self.sum()
}
#[inline]
fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() }
fn partial_kmer_counts(&self) -> Array1<u64> {
self.count_nonzero()
}
}
impl CountPartials for PersistentSparseCompactIntMatrix {
@@ -278,7 +305,9 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
fn partial_bray(&self) -> Array2<u64> {
let mut m = self.row_major_pairwise(|_, a, _, b| (a as u64).min(b as u64));
let sum = self.sum();
for i in 0..self.n_cols() { m[[i, i]] = sum[i]; }
for i in 0..self.n_cols() {
m[[i, i]] = sum[i];
}
m
}
@@ -329,7 +358,8 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
// left to patch.
return BitPartials::partial_jaccard(&self.support);
}
let mut inter = self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64);
let mut inter =
self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64);
let mut union = Array2::<u64>::zeros((n, n));
for i in 0..n {
for j in 0..n {
@@ -363,7 +393,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
});
let sum = self.sum();
for i in 0..self.n_cols() {
m[[i, i]] = if global[i] > 0 { sum[i] as f64 / global[i] as f64 } else { 0.0 };
m[[i, i]] = if global[i] > 0 {
sum[i] as f64 / global[i] as f64
} else {
0.0
};
}
m
}
@@ -382,9 +416,21 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
if i != j {
let sa = global[i] as f64;
let sb = global[j] as f64;
let sq_a = if sa > 0.0 { sq[i] as f64 / (sa * sa) } else { 0.0 };
let sq_b = if sb > 0.0 { sq[j] as f64 / (sb * sb) } else { 0.0 };
let cross = if sa > 0.0 && sb > 0.0 { dot[[i, j]] / (sa * sb) } else { 0.0 };
let sq_a = if sa > 0.0 {
sq[i] as f64 / (sa * sa)
} else {
0.0
};
let sq_b = if sb > 0.0 {
sq[j] as f64 / (sb * sb)
} else {
0.0
};
let cross = if sa > 0.0 && sb > 0.0 {
dot[[i, j]] / (sa * sb)
} else {
0.0
};
m[[i, j]] = sq_a + sq_b - 2.0 * cross;
}
}
@@ -409,7 +455,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
let sb = global[j] as f64;
let pa_sum = if sa > 0.0 { sum[i] as f64 / sa } else { 0.0 };
let pb_sum = if sb > 0.0 { sum[j] as f64 / sb } else { 0.0 };
let cross = if sa > 0.0 && sb > 0.0 { sqrt_dot[[i, j]] / (sa * sb).sqrt() } else { 0.0 };
let cross = if sa > 0.0 && sb > 0.0 {
sqrt_dot[[i, j]] / (sa * sb).sqrt()
} else {
0.0
};
m[[i, j]] = pa_sum + pb_sum - 2.0 * cross;
}
}
@@ -461,7 +511,6 @@ impl PersistentSparseCompactIntMatrixBuilder {
self.support.push_row(cols);
}
/// Builds a sparse matrix from an already-built dense
/// [`PersistentCompactIntMatrix`] (`Columnar` or `Packed`) — same
/// batched-transpose shape as
@@ -475,7 +524,7 @@ impl PersistentSparseCompactIntMatrixBuilder {
/// pairs are appended to its own buffer in strictly ascending column
/// order "for free", matching `push_row`'s ascending-`cols` contract
/// without an explicit sort.
pub fn build_from_dense(dense: &PersistentCompactIntMatrix, dir: &Path) -> io::Result<Self> {
pub fn build_from_dense(dense: &PersistentIntMatrix, dir: &Path) -> io::Result<Self> {
let n = dense.n();
let n_cols = dense.n_cols();
let mut builder = Self::new(n, n_cols, dir)?;
@@ -500,23 +549,34 @@ impl PersistentSparseCompactIntMatrixBuilder {
}
pub fn close(self) -> io::Result<()> {
let Self { support, dir, singleton_values, multi_values, multi_row_offsets } = self;
let Self {
support,
dir,
singleton_values,
multi_values,
multi_row_offsets,
} = self;
support.close()?;
let mut sv = PersistentCompactIntVecBuilder::new(singleton_values.len(), &singleton_values_path(&dir))?;
let mut sv = PersistentCompactIntVecBuilder::new(
singleton_values.len(),
&singleton_values_path(&dir),
)?;
for (i, &v) in singleton_values.iter().enumerate() {
sv.set(i, v);
}
sv.close()?;
let mut mv = PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?;
let mut mv =
PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?;
for (i, &v) in multi_values.iter().enumerate() {
mv.set(i, v);
}
mv.close()?;
let universe = multi_values.len() as u64 + 1;
let mut mo = EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?;
let mut mo =
EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?;
for &o in &multi_row_offsets {
mo.push(o);
}
@@ -544,7 +604,7 @@ impl PersistentSparseCompactIntMatrixBuilder {
/// half-written one). Idempotent — a no-op if `singleton_values.pciv`
/// already exists.
pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
use crate::intmatrix::{col_path, ColumnarCompactIntMatrix, PackedCompactIntMatrix};
use crate::intmatrix::{ColumnarCompactIntMatrix, PackedCompactIntMatrix, col_path};
if is_present(dir) {
return Ok(());
@@ -552,12 +612,12 @@ pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
let packed_path = dir.join("matrix.pcmx");
if packed_path.exists() {
let dense = PersistentCompactIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?);
let dense = PersistentIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?);
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
drop(dense);
fs::remove_file(&packed_path)?;
} else {
let dense = PersistentCompactIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?);
let dense = PersistentIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?);
let n_cols = dense.n_cols();
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
drop(dense);
src/obicompactvec/src/tests/colgroup.rs
+33 -20
View File
@@ -1,25 +1,26 @@
use tempfile::tempdir;
use crate::{
ColGroup, MatrixGroupOps,
PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentIntMatrix, PersistentCompactIntMatrixBuilder,
};
use crate::{PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder};
// ── helpers ───────────────────────────────────────────────────────────────────
fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
let n = cols.first().map_or(0, |c| c.len());
let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
for &col in cols {
let mut cb = b.add_col().unwrap();
for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
for (slot, &v) in col.iter().enumerate() {
cb.set(slot, v);
}
cb.close().unwrap();
}
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
(dir, m)
}
@@ -30,7 +31,9 @@ fn make_bit_matrix(cols: &[&[bool]]) -> (tempfile::TempDir, PersistentBitMatrix)
let mut b = PersistentBitMatrixBuilder::new(n, &presence).unwrap();
for &col in cols {
let mut cb = b.add_col().unwrap();
for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
for (slot, &v) in col.iter().enumerate() {
cb.set(slot, v);
}
cb.close().unwrap();
}
b.close().unwrap();
@@ -115,9 +118,13 @@ fn mask_with_zeros_selected_slots() {
// count vec [10, 20, 30, 40], mask [T, F, T, F] → [10, 0, 30, 0]
let dir = tempdir().unwrap();
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
v.set(0, 10); v.set(1, 20); v.set(2, 30); v.set(3, 40);
v.set(0, 10);
v.set(1, 20);
v.set(2, 30);
v.set(3, 40);
let mut mask = PersistentBitVecBuilder::new(4, &dir.path().join("m.pbiv")).unwrap();
mask.set(0, true); mask.set(2, true);
mask.set(0, true);
mask.set(2, true);
v.mask_with(mask.view());
v.close().unwrap();
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
@@ -132,9 +139,12 @@ fn mask_with_overflow_slot_zeroed() {
// overflow slot (value 500) masked out → removed from overflow, primary=0
let dir = tempdir().unwrap();
let mut v = PersistentCompactIntVecBuilder::new(3, &dir.path().join("v.pciv")).unwrap();
v.set(0, 10); v.set(1, 500); v.set(2, 5);
v.set(0, 10);
v.set(1, 500);
v.set(2, 5);
let mut mask = PersistentBitVecBuilder::new(3, &dir.path().join("m.pbiv")).unwrap();
mask.set(0, true); mask.set(2, true); // slot 1 masked out
mask.set(0, true);
mask.set(2, true); // slot 1 masked out
v.mask_with(mask.view());
v.close().unwrap();
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
@@ -142,14 +152,20 @@ fn mask_with_overflow_slot_zeroed() {
assert_eq!(r.get(1), 0);
assert_eq!(r.get(2), 5);
let ov: Vec<_> = r.view().overflow_entries().collect();
assert!(ov.is_empty(), "overflow entry for masked-out slot should be gone");
assert!(
ov.is_empty(),
"overflow entry for masked-out slot should be gone"
);
}
#[test]
fn mask_with_all_ones_is_noop() {
let dir = tempdir().unwrap();
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
v.set(0, 300); v.set(1, 1); v.set(2, 0); v.set(3, 42);
v.set(0, 300);
v.set(1, 1);
v.set(2, 0);
v.set(3, 42);
let mask = PersistentBitVecBuilder::new_ones(4, &dir.path().join("m.pbiv")).unwrap();
v.mask_with(mask.view());
v.close().unwrap();
@@ -167,9 +183,9 @@ fn bit_partial_group_presence_count() {
// col0=[T,F,T,F], col1=[T,T,F,F], col2=[F,T,T,F]
// group {0,1,2}: counts = [2, 2, 2, 0]
let (_d, m) = make_bit_matrix(&[
&[true, false, true, false],
&[true, true, false, false],
&[false,true, true, false],
&[true, false, true, false],
&[true, true, false, false],
&[false, true, true, false],
]);
let g = ColGroup::new("g", vec![0, 1, 2]);
let result = m.partial_group_presence_count(&g, 1).unwrap();
@@ -184,10 +200,7 @@ fn bit_partial_group_presence_count() {
#[test]
fn bit_partial_group_any() {
// col0=[T,F,F], col1=[F,F,T], group {0,1}: any = [T, F, T]
let (_d, m) = make_bit_matrix(&[
&[true, false, false],
&[false, false, true],
]);
let (_d, m) = make_bit_matrix(&[&[true, false, false], &[false, false, true]]);
let g = ColGroup::new("g", vec![0, 1]);
let result = m.partial_group_any(&g, 1).unwrap();
assert_eq!(result.get(0), true);
src/obicompactvec/src/tests/intmatrix.rs
+109 -45
View File
@@ -1,9 +1,12 @@
use tempfile::tempdir;
use crate::{pack_compact_int_matrix, pack_sparse_compact_int_matrix, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder, StorageKind};
use crate::traits::CountPartials;
use crate::{
PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder,
PersistentIntMatrix, StorageKind, pack_compact_int_matrix, pack_sparse_compact_int_matrix,
};
fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
let n = cols.first().map_or(0, |c| c.len());
let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
@@ -15,7 +18,7 @@ fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatri
cb.close().unwrap();
}
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
(dir, m)
}
@@ -31,7 +34,7 @@ fn single_col_roundtrip() {
col.close().unwrap();
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 1);
assert_eq!(m.n(), 4);
assert_eq!(&*m.row(0), &[10u32]);
@@ -45,14 +48,18 @@ fn two_cols_roundtrip() {
let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(3, &dir.path().join("counts")).unwrap();
let mut col0 = b.add_col().unwrap();
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3);
col0.set(0, 1);
col0.set(1, 2);
col0.set(2, 3);
col0.close().unwrap();
let mut col1 = b.add_col().unwrap();
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30);
col1.set(0, 10);
col1.set(1, 20);
col1.set(2, 30);
col1.close().unwrap();
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 2);
assert_eq!(&*m.row(0), &[1u32, 10]);
assert_eq!(&*m.row(1), &[2u32, 20]);
@@ -66,7 +73,9 @@ fn resume_continues_n_cols_and_appends_columns() {
let mut b = PersistentCompactIntMatrixBuilder::new(3, &counts).unwrap();
let mut col0 = b.add_col().unwrap();
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3);
col0.set(0, 1);
col0.set(1, 2);
col0.set(2, 3);
col0.close().unwrap();
b.close().unwrap();
@@ -74,11 +83,13 @@ fn resume_continues_n_cols_and_appends_columns() {
assert_eq!(resumed.n(), 3);
assert_eq!(resumed.n_cols(), 1);
let mut col1 = resumed.add_col().unwrap();
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30);
col1.set(0, 10);
col1.set(1, 20);
col1.set(2, 30);
col1.close().unwrap();
resumed.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 2);
assert_eq!(&*m.row(0), &[1u32, 10]);
assert_eq!(&*m.row(1), &[2u32, 20]);
@@ -90,17 +101,21 @@ fn resume_twice_keeps_appending() {
let dir = tempdir().unwrap();
let counts = dir.path().join("counts");
PersistentCompactIntMatrixBuilder::new(2, &counts).unwrap().close().unwrap();
PersistentCompactIntMatrixBuilder::new(2, &counts)
.unwrap()
.close()
.unwrap();
for v in [1u32, 2, 3] {
let mut b = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap();
let mut col = b.add_col().unwrap();
col.set(0, v); col.set(1, v * 10);
col.set(0, v);
col.set(1, v * 10);
col.close().unwrap();
b.close().unwrap();
}
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 3);
assert_eq!(&*m.row(0), &[1u32, 2, 3]);
assert_eq!(&*m.row(1), &[10u32, 20, 30]);
@@ -111,11 +126,12 @@ fn col_accessor() {
let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(2, &dir.path().join("counts")).unwrap();
let mut col0 = b.add_col().unwrap();
col0.set(0, 5); col0.set(1, 7);
col0.set(0, 5);
col0.set(1, 7);
col0.close().unwrap();
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.col(0).get(0), 5);
assert_eq!(m.col(0).get(1), 7);
}
@@ -126,7 +142,7 @@ fn zero_cols_roundtrip() {
let b = PersistentCompactIntMatrixBuilder::new(10, &dir.path().join("counts")).unwrap();
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 0);
assert_eq!(m.n(), 10);
}
@@ -139,7 +155,9 @@ fn bray_dist_matrix_symmetry_and_diagonal() {
let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]);
let dm = m.bray_dist_matrix();
let n = m.n_cols();
for i in 0..n { assert_eq!(dm[[i, i]], 0.0, "diagonal"); }
for i in 0..n {
assert_eq!(dm[[i, i]], 0.0, "diagonal");
}
for i in 0..n {
for j in 0..n {
assert!((dm[[i, j]] - dm[[j, i]]).abs() < 1e-12, "symmetry");
@@ -174,7 +192,7 @@ fn jaccard_dist_matrix_values_match_pairwise() {
#[test]
fn partial_bray_dist_matrix_consistent() {
let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]);
let sum_min = m.partial_bray_dist_matrix();
let sum_min = m.partial_bray_dist_matrix();
let col_sums = m.sum();
let n = m.n_cols();
@@ -189,7 +207,11 @@ fn partial_bray_dist_matrix_consistent() {
for i in 0..n {
for j in i + 1..n {
let denom = col_sums[i] + col_sums[j];
let dist = if denom == 0 { 0.0 } else { 1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64 };
let dist = if denom == 0 {
0.0
} else {
1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64
};
let expected = m.col(i).bray_dist(m.col(j));
assert!((dist - expected).abs() < 1e-12, "[{i},{j}]");
}
@@ -249,13 +271,18 @@ fn partial_relfreq_bray_matches_full() {
let (_d, m) = make_matrix(&[&[1, 0, 2], &[0, 1, 1], &[1, 1, 0]]);
let col_sums = m.sum();
let partial = m.partial_relfreq_bray_dist_matrix(&col_sums);
let full = m.relfreq_bray_dist_matrix();
let full = m.relfreq_bray_dist_matrix();
let n = m.n_cols();
// partial[i,j] = sum_min_relfreq; full[i,j] = 1 - sum_min_relfreq (off-diagonal only)
for i in 0..n {
for j in 0..n {
if i == j { continue; }
assert!((partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12, "[{i},{j}]");
if i == j {
continue;
}
assert!(
(partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12,
"[{i},{j}]"
);
}
}
}
@@ -265,12 +292,15 @@ fn partial_relfreq_euclidean_matches_full() {
let (_d, m) = make_matrix(&[&[3, 0], &[0, 4], &[1, 1]]);
let col_sums = m.sum();
let partial = m.partial_relfreq_euclidean_dist_matrix(&col_sums);
let full = m.relfreq_euclidean_dist_matrix();
let full = m.relfreq_euclidean_dist_matrix();
let n = m.n_cols();
for i in 0..n {
for j in 0..n {
// partial = squared euclidean; full = sqrt(partial)
assert!((partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12, "[{i},{j}]");
assert!(
(partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12,
"[{i},{j}]"
);
}
}
}
@@ -280,12 +310,15 @@ fn partial_hellinger_matches_full() {
let (_d, m) = make_matrix(&[&[3, 0], &[0, 4], &[1, 1]]);
let col_sums = m.sum();
let partial = m.partial_hellinger_euclidean_dist_matrix(&col_sums);
let full = m.hellinger_dist_matrix();
let full = m.hellinger_dist_matrix();
let n = m.n_cols();
for i in 0..n {
for j in 0..n {
// partial / sqrt(2) gives the Hellinger distance
assert!((partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12, "[{i},{j}]");
assert!(
(partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12,
"[{i},{j}]"
);
}
}
}
@@ -295,7 +328,7 @@ fn col_view_packed_values() {
// Build Columnar with overflow values (≥ 255), pack, reopen as Packed, exercise col_view().
let (dir, _col) = make_matrix(&[&[10, 300, 500], &[200, 50, 1000]]);
pack_compact_int_matrix(&dir.path().join("counts")).unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
// col 0: [10, 300, 500] — two overflow slots
let v0 = m.col_view(0);
@@ -326,10 +359,10 @@ fn col_view_packed_matches_columnar() {
// Re-build in a separate dir so we can pack without touching m_col's files.
let (dir_pack, _) = make_matrix(data);
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap();
let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap();
for c in 0..data.len() {
let col_ref = m_col.col(c);
let col_ref = m_col.col(c);
let col_view = m_pack.col_view(c);
assert_eq!(col_view.len(), col_ref.len());
for s in 0..col_ref.len() {
@@ -337,7 +370,7 @@ fn col_view_packed_matches_columnar() {
}
assert_eq!(col_view.sum(), col_ref.sum(), "col={c} sum");
let mut ov_view: Vec<(usize, u32)> = col_view.overflow_entries().collect();
let mut ov_ref: Vec<(usize, u32)> = col_ref.view().overflow_entries().collect();
let mut ov_ref: Vec<(usize, u32)> = col_ref.view().overflow_entries().collect();
ov_view.sort_unstable_by_key(|&(s, _)| s);
ov_ref.sort_unstable_by_key(|&(s, _)| s);
assert_eq!(ov_view, ov_ref, "col={c} overflow_entries");
@@ -355,7 +388,7 @@ fn nonzero_iter_matches_row() {
let (dir_col, m_col) = make_matrix(data);
let (dir_pack, _) = make_matrix(data);
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap();
let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap();
let slots = [4usize, 0, 3, 1];
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
@@ -390,7 +423,7 @@ fn sparse_roundtrip_matches_columnar() {
let (dir_col, m_col) = make_matrix(data);
let (dir_sparse, _) = make_matrix(data);
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
assert_eq!(m_sparse.n(), m_col.n());
@@ -441,7 +474,7 @@ fn sparse_count_partials_match_dense() {
let (dir_col, m_col) = make_matrix(data);
let (dir_sparse, _) = make_matrix(data);
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
let n = m_col.n_cols();
@@ -461,7 +494,11 @@ fn sparse_count_partials_match_dense() {
let eucl_sparse = m_sparse.partial_euclidean_dist_matrix();
for i in 0..n {
for j in 0..n {
close(eucl_col[[i, j]], eucl_sparse[[i, j]], &format!("partial_euclidean[{i},{j}]"));
close(
eucl_col[[i, j]],
eucl_sparse[[i, j]],
&format!("partial_euclidean[{i},{j}]"),
);
}
}
@@ -469,9 +506,16 @@ fn sparse_count_partials_match_dense() {
// (support shortcut), and >1 (general row-major path).
for threshold in [0u32, 1, 2, 3] {
let (inter_col, union_col) = m_col.partial_threshold_jaccard_dist_matrix(threshold);
let (inter_sparse, union_sparse) = m_sparse.partial_threshold_jaccard_dist_matrix(threshold);
assert_eq!(inter_col, inter_sparse, "partial_threshold_jaccard inter, threshold={threshold}");
assert_eq!(union_col, union_sparse, "partial_threshold_jaccard union, threshold={threshold}");
let (inter_sparse, union_sparse) =
m_sparse.partial_threshold_jaccard_dist_matrix(threshold);
assert_eq!(
inter_col, inter_sparse,
"partial_threshold_jaccard inter, threshold={threshold}"
);
assert_eq!(
union_col, union_sparse,
"partial_threshold_jaccard union, threshold={threshold}"
);
}
// partial_relfreq_bray
@@ -479,7 +523,11 @@ fn sparse_count_partials_match_dense() {
let rfb_sparse = m_sparse.partial_relfreq_bray_dist_matrix(&global);
for i in 0..n {
for j in 0..n {
close(rfb_col[[i, j]], rfb_sparse[[i, j]], &format!("partial_relfreq_bray[{i},{j}]"));
close(
rfb_col[[i, j]],
rfb_sparse[[i, j]],
&format!("partial_relfreq_bray[{i},{j}]"),
);
}
}
@@ -488,7 +536,11 @@ fn sparse_count_partials_match_dense() {
let rfe_sparse = m_sparse.partial_relfreq_euclidean_dist_matrix(&global);
for i in 0..n {
for j in 0..n {
close(rfe_col[[i, j]], rfe_sparse[[i, j]], &format!("partial_relfreq_euclidean[{i},{j}]"));
close(
rfe_col[[i, j]],
rfe_sparse[[i, j]],
&format!("partial_relfreq_euclidean[{i},{j}]"),
);
}
}
@@ -497,7 +549,11 @@ fn sparse_count_partials_match_dense() {
let hel_sparse = m_sparse.partial_hellinger_euclidean_dist_matrix(&global);
for i in 0..n {
for j in 0..n {
close(hel_col[[i, j]], hel_sparse[[i, j]], &format!("partial_hellinger[{i},{j}]"));
close(
hel_col[[i, j]],
hel_sparse[[i, j]],
&format!("partial_hellinger[{i},{j}]"),
);
}
}
@@ -514,7 +570,7 @@ fn sparse_roundtrip_from_packed() {
let (dir_sparse, _) = make_matrix(data);
pack_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
for slot in 0..m_col.n() {
assert_eq!(&*m_sparse.row(slot), &*m_col.row(slot), "slot={slot}");
@@ -540,7 +596,10 @@ fn partial_relfreq_bray_additive_across_split() {
let n = m_full.n_cols();
for i in 0..n {
for j in 0..n {
assert!((combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12, "[{i},{j}]");
assert!(
(combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12,
"[{i},{j}]"
);
}
}
}
@@ -548,10 +607,12 @@ fn partial_relfreq_bray_additive_across_split() {
// ── collect_slots_values tests ────────────────────────────────────────────────────────────
fn make_pciv(counts: &[u32]) -> (tempfile::TempDir, PersistentCompactIntVec) {
let dir = tempdir().unwrap();
let dir = tempdir().unwrap();
let path = dir.path().join("c.pciv");
let mut b = PersistentCompactIntVecBuilder::new(counts.len(), &path).unwrap();
for (i, &v) in counts.iter().enumerate() { b.set(i, v); }
for (i, &v) in counts.iter().enumerate() {
b.set(i, v);
}
b.close().unwrap();
let r = PersistentCompactIntVec::open(&path).unwrap();
(dir, r)
@@ -592,7 +653,10 @@ fn pciv_collect_slots_values_empty() {
fn pciv_collect_slots_values_out_of_bounds_panics() {
let (_dir, v) = make_pciv(&[10u32, 20]);
let result = std::panic::catch_unwind(|| v.collect_slots_values(&[0, 2]));
assert!(result.is_err(), "collect_slots_values should panic on out-of-bounds slot");
assert!(
result.is_err(),
"collect_slots_values should panic on out-of-bounds slot"
);
}
// IntSliceView collect_slots_values (same logic, exercised through the view)
src/obicompactvec/src/tests/sparse_intmatrix.rs
+52 -52
View File
@@ -1,13 +1,14 @@
use tempfile::tempdir;
use crate::{
pack_compact_int_matrix, pack_sparse_compact_int_matrix, ColumnWeights, PersistentCompactIntMatrix,
PersistentCompactIntMatrixBuilder, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
ColumnWeights, PersistentIntMatrix, PersistentCompactIntMatrixBuilder,
PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
pack_compact_int_matrix, pack_sparse_compact_int_matrix,
};
/// Builds a dense `PersistentCompactIntMatrix` from column-major `u32` data
/// — mirrors `tests/intmatrix.rs`'s own `make_matrix` helper.
fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
let n = cols.first().map_or(0, |c| c.len());
let dir = tempdir().unwrap();
let counts_dir = dir.path().join("counts");
@@ -20,17 +21,21 @@ fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix
cb.close().unwrap();
}
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
(dir, m)
}
/// Builds a sparse int matrix directly from row-major `u32` data (one
/// slice per row, `n_cols` values each) — mirrors `tests/sparse.rs`'s
/// `make_sparse` helper.
fn make_sparse(rows: &[&[u32]], n_cols: usize) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) {
fn make_sparse(
rows: &[&[u32]],
n_cols: usize,
) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) {
let dir = tempdir().unwrap();
let sparse_dir = dir.path().join("sparse");
let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap();
let mut b =
PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap();
let mut cols = Vec::new();
let mut values = Vec::new();
for row in rows {
@@ -54,11 +59,11 @@ fn basic_roundtrip_singletons_and_multi() {
// shares its *support* (columns {0,1}) with another but has different
// values — the whole point of not deduplicating values.
let rows: Vec<&[u32]> = vec![
&[5, 0, 0, 0], // singleton: genome 0, value 5
&[0, 0, 3, 0], // singleton: genome 2, value 3
&[7, 2, 0, 0], // multi: {0:7, 1:2}
&[0, 0, 4, 9], // multi: {2:4, 3:9}
&[1, 6, 0, 0], // multi: same support {0,1} as row 2, different values
&[5, 0, 0, 0], // singleton: genome 0, value 5
&[0, 0, 3, 0], // singleton: genome 2, value 3
&[7, 2, 0, 0], // multi: {0:7, 1:2}
&[0, 0, 4, 9], // multi: {2:4, 3:9}
&[1, 6, 0, 0], // multi: same support {0,1} as row 2, different values
];
let (_dir, m) = make_sparse(&rows, 4);
assert_eq!(m.n(), 5);
@@ -70,11 +75,7 @@ fn basic_roundtrip_singletons_and_multi() {
#[test]
fn get_matches_row() {
let rows: Vec<&[u32]> = vec![
&[3, 0, 5],
&[0, 8, 0],
&[1, 2, 4],
];
let rows: Vec<&[u32]> = vec![&[3, 0, 5], &[0, 8, 0], &[1, 2, 4]];
let (_dir, m) = make_sparse(&rows, 3);
for (slot, row) in rows.iter().enumerate() {
for (c, &expected) in row.iter().enumerate() {
@@ -85,11 +86,7 @@ fn get_matches_row() {
#[test]
fn fill_row_matches_row() {
let rows: Vec<&[u32]> = vec![
&[9, 0, 2],
&[0, 4, 0],
&[1, 1, 1],
];
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]];
let (_dir, m) = make_sparse(&rows, 3);
let mut buf = vec![0u32; 3];
for slot in 0..3 {
@@ -120,11 +117,7 @@ fn fill_sub_matrix_matches_row() {
#[test]
fn nonzero_iter_matches_row() {
let rows: Vec<&[u32]> = vec![
&[9, 0, 2],
&[0, 4, 0],
&[1, 1, 1],
];
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]];
let (_dir, m) = make_sparse(&rows, 3);
let slots = [2usize, 0, 1];
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
@@ -143,12 +136,7 @@ fn nonzero_iter_matches_row() {
#[test]
fn sum_and_count_nonzero_match_naive() {
let rows: Vec<&[u32]> = vec![
&[9, 0, 2],
&[0, 4, 0],
&[1, 1, 1],
&[0, 0, 6],
];
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1], &[0, 0, 6]];
let (_dir, m) = make_sparse(&rows, 3);
let mut expected_sum = [0u64; 3];
let mut expected_count = [0u64; 3];
@@ -193,14 +181,21 @@ fn reopen_after_close_matches_original() {
let rows: Vec<Vec<u32>> = (0..500)
.map(|i| {
(0..37)
.map(|c| if (i * 7 + c * 3) % 11 == 0 { ((i + c) % 250 + 1) as u32 } else { 0 })
.map(|c| {
if (i * 7 + c * 3) % 11 == 0 {
((i + c) % 250 + 1) as u32
} else {
0
}
})
.collect()
})
.collect();
let dir = tempdir().unwrap();
let sparse_dir = dir.path().join("sparse");
{
let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap();
let mut b =
PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap();
let mut cols = Vec::new();
let mut values = Vec::new();
for row in &rows {
@@ -229,10 +224,7 @@ fn reopen_after_close_matches_original() {
fn overflow_values_roundtrip() {
// Values >= 255 exercise the PersistentCompactIntVec overflow path in
// both the singleton and multi value streams.
let rows: Vec<&[u32]> = vec![
&[1000, 0, 0],
&[0, 50_000, 300],
];
let rows: Vec<&[u32]> = vec![&[1000, 0, 0], &[0, 50_000, 300]];
let (_dir, m) = make_sparse(&rows, 3);
for (slot, &expected) in rows.iter().enumerate() {
assert_eq!(&*m.row(slot), expected, "row {slot}");
@@ -246,14 +238,13 @@ fn overflow_values_roundtrip() {
/// `tests::sparse::pack_sparse_bit_matrix_transposes_columnar_directly`.
#[test]
fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
let cols: &[&[u32]] = &[
&[9, 0, 2, 0],
&[0, 4, 0, 7],
&[1, 1, 300, 1],
];
let cols: &[&[u32]] = &[&[9, 0, 2, 0], &[0, 4, 0, 7], &[1, 1, 300, 1]];
let (dir, dense) = make_dense(cols);
let counts_dir = dir.path().join("counts");
assert!(!counts_dir.join("matrix.pcmx").exists(), "still Columnar before packing");
assert!(
!counts_dir.join("matrix.pcmx").exists(),
"still Columnar before packing"
);
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
let n_cols = dense.n_cols();
@@ -262,9 +253,18 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
assert!(counts_dir.join("singleton_values.pciv").exists());
assert!(!counts_dir.join("meta.json").exists(), "columnar meta.json cleaned up");
assert!(!counts_dir.join("col_000000.pciv").exists(), "columnar column files cleaned up");
assert!(!counts_dir.join("matrix.pcmx").exists(), "never materialised");
assert!(
!counts_dir.join("meta.json").exists(),
"columnar meta.json cleaned up"
);
assert!(
!counts_dir.join("col_000000.pciv").exists(),
"columnar column files cleaned up"
);
assert!(
!counts_dir.join("matrix.pcmx").exists(),
"never materialised"
);
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
assert_eq!(sparse.n(), expected_rows.len());
@@ -282,10 +282,7 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
/// it's still used as the transpose source and cleaned up afterward.
#[test]
fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
let cols: &[&[u32]] = &[
&[9, 0, 2],
&[0, 4, 300],
];
let cols: &[&[u32]] = &[&[9, 0, 2], &[0, 4, 300]];
let (dir, dense) = make_dense(cols);
let counts_dir = dir.path().join("counts");
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
@@ -296,7 +293,10 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
assert!(counts_dir.join("matrix.pcmx").exists());
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
assert!(!counts_dir.join("matrix.pcmx").exists(), "packed intermediate cleaned up");
assert!(
!counts_dir.join("matrix.pcmx").exists(),
"packed intermediate cleaned up"
);
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
assert_eq!(sparse.n_cols(), n_cols);
src/obikindex/src/layer/content_layer.rs
+7 -4
View File
@@ -26,7 +26,7 @@ use std::path::{Path, PathBuf};
use crate::layer::utils::LAYERNAME_SUFFIX;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
use obikseq::CanonicalKmer;
use crate::index::error::OKIResult;
@@ -63,7 +63,7 @@ pub enum KmerLayer {
Count {
dir: PathBuf,
id: usize,
layer: TypedLayer<PersistentCompactIntMatrix>,
layer: TypedLayer<PersistentIntMatrix>,
},
Presence {
dir: PathBuf,
@@ -99,7 +99,7 @@ impl KmerLayer {
already_open => return Ok(already_open),
};
if dir.join(COUNTS_DIR).exists() {
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(&dir)?;
let layer = TypedLayer::<PersistentIntMatrix>::open(&dir)?;
Ok(KmerLayer::Count { dir, id, layer })
} else {
let layer = TypedLayer::<PersistentBitMatrix>::open(&dir)?;
@@ -221,7 +221,10 @@ impl KmerLayer {
/// format-agnostic column-major fetch, delegating to whichever matrix
/// this layer actually holds (see `obicompactvec::PersistentBitMatrix`/
/// `PersistentCompactIntMatrix::nonzero_iter`).
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
match self {
KmerLayer::Presence { layer, .. } => layer.nonzero_iter(slots),
KmerLayer::Count { layer, .. } => layer.nonzero_iter(slots),
src/obikindex/src/layer/tests/typed_layer.rs
+71 -30
View File
@@ -1,6 +1,6 @@
use super::*;
use obicompactvec::PersistentSparseBitMatrixBuilder;
use obikseq::{set_k, Unitig};
use obikseq::{Unitig, set_k};
use obiskio::DEFAULT_BLOCK_BITS;
use tempfile::tempdir;
@@ -13,7 +13,8 @@ fn write_unitigs(dir: &Path, seqs: &[&[u8]]) {
}
fn all_canonical_kmers(dir: &Path) -> Vec<CanonicalKmer> {
UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE)).unwrap()
UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE))
.unwrap()
.iter_indexed_canonical_kmers()
.map(|(kmer, _, _)| kmer)
.collect()
@@ -34,13 +35,17 @@ fn canonical_kmer_iter_matches_reader() {
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAAACGT", b"TTTTGCA"]);
let from_iter: Vec<CanonicalKmer> = obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE))
.unwrap()
.collect();
let from_iter: Vec<CanonicalKmer> =
obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE))
.unwrap()
.collect();
let from_reader: Vec<CanonicalKmer> = all_canonical_kmers(dir.path());
assert_eq!(from_iter.len(), from_reader.len(), "different kmer counts");
assert_eq!(from_iter, from_reader, "CanonicalKmerIter and UnitigFileReader disagree");
assert_eq!(
from_iter, from_reader,
"CanonicalKmerIter and UnitigFileReader disagree"
);
}
// ── Generic `TypedLayer<D>` over dense vs. sparse presence storage ───────────────
@@ -64,9 +69,14 @@ fn presence_layer_generic_over_sparse_matches_dense() {
// kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be
// biologically meaningful, just the same on both sides of the
// dense/sparse comparison below.
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
}).unwrap();
TypedLayer::<PersistentBitMatrix>::build_presence(
dir.path(),
DEFAULT_BLOCK_BITS,
&mode,
n_genomes,
|kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0,
)
.unwrap();
let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
assert!(dense_layer.n_cols() >= 1);
@@ -76,10 +86,13 @@ fn presence_layer_generic_over_sparse_matches_dense() {
// same directory, distinct filenames — see
// `DevDocMD/architecture/siblings.md`'s sparse-matrix section).
let dense_matrix = obicompactvec::PersistentBitMatrix::open(dir.path()).unwrap();
PersistentSparseBitMatrixBuilder::build_from_dense(&dense_matrix, &dir.path().join(PRESENCE_DIR))
.unwrap()
.close()
.unwrap();
PersistentSparseBitMatrixBuilder::build_from_dense(
&dense_matrix,
&dir.path().join(PRESENCE_DIR),
)
.unwrap()
.close()
.unwrap();
let sparse_layer = TypedLayer::<PersistentSparseBitMatrix>::open(dir.path()).unwrap();
@@ -88,7 +101,10 @@ fn presence_layer_generic_over_sparse_matches_dense() {
let n = dense_layer.n();
assert_eq!(n, sparse_layer.n());
let slots: Vec<usize> = (0..n).collect();
assert_eq!(dense_layer.sub_matrix(&slots), sparse_layer.sub_matrix(&slots));
assert_eq!(
dense_layer.sub_matrix(&slots),
sparse_layer.sub_matrix(&slots)
);
// The matrix-agnostic, MPHF-only surface (from the generic
// `impl<D: LayerData> TypedLayer<D>`) must also agree: same kmer set,
@@ -124,15 +140,25 @@ fn count_layer_transparently_reads_sparse_after_pack() {
// Deterministic, arbitrary count function — same shape as the presence
// test's arbitrary predicate, just producing a small count instead of a
// bool.
TypedLayer::<()>::build_with_matrix(dir.path(), DEFAULT_BLOCK_BITS, &mode, false, n_genomes, |kmer| {
(0..n_genomes)
.map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32)
.collect()
})
TypedLayer::<()>::build_with_matrix(
dir.path(),
DEFAULT_BLOCK_BITS,
&mode,
false,
n_genomes,
|kmer| {
(0..n_genomes)
.map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32)
.collect()
},
)
.unwrap();
let dense_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
assert_eq!(dense_layer.storage_kind(), obicompactvec::StorageKind::Columnar);
let dense_layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
assert_eq!(
dense_layer.storage_kind(),
obicompactvec::StorageKind::Columnar
);
let n = dense_layer.n();
let slots: Vec<usize> = (0..n).collect();
let dense_sub = dense_layer.sub_matrix(&slots);
@@ -146,8 +172,11 @@ fn count_layer_transparently_reads_sparse_after_pack() {
// in production.
obicompactvec::pack_sparse_compact_int_matrix(&dir.path().join(COUNTS_DIR)).unwrap();
let sparse_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
assert_eq!(sparse_layer.storage_kind(), obicompactvec::StorageKind::Sparse);
let sparse_layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
assert_eq!(
sparse_layer.storage_kind(),
obicompactvec::StorageKind::Sparse
);
assert_eq!(sparse_layer.n(), n);
assert_eq!(sparse_layer.n_cols(), dense_layer.n_cols());
assert_eq!(sparse_layer.sub_matrix(&slots), dense_sub);
@@ -167,13 +196,20 @@ fn count_layer_reports_count_content_and_columnar_storage() {
set_k(4);
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAAACGT"]);
TypedLayer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
.unwrap();
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
TypedLayer::<PersistentIntMatrix>::build(dir.pat
h(), DEFAUL
T_BLOCK_BITS, &Index
Mode::Exact, |_| 1)
,
).unwrap();
let layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
assert_eq!(layer.content(), LayerContent::Count);
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
assert_eq!(layer.evidence_kind(), crate::layer::mphf_layer::EvidenceKind::Exact);
assert_eq!(
layer.evidence_kind(),
crate::layer::mphf_layer::EvidenceKind::Exact
);
}
#[test]
@@ -181,9 +217,14 @@ fn presence_layer_reports_presence_content_and_columnar_storage() {
set_k(4);
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
}).unwrap();
TypedLayer::<PersistentBitMatrix>::build_presence(
dir.path(),
DEFAULT_BLOCK_BITS,
&IndexMode::Exact,
2,
|kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0,
)
.unwrap();
let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
assert_eq!(layer.content(), LayerContent::Presence);
src/obikindex/src/layer/typed_layer.rs
+17 -11
View File
@@ -1,7 +1,7 @@
// use crate::layer::utils::layer_dir;
use obicompactvec::{
BinaryMatrix, ColBuilder, MatrixBuilder, PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentSparseBitMatrix,
PersistentCompactIntMatrixBuilder, PersistentIntMatrix, PersistentSparseBitMatrix,
};
use obikseq::CanonicalKmer;
use obiskio::{UnitigFileReader, UnitigFileWriter};
@@ -47,10 +47,10 @@ impl LayerData for () {
fn read(&self, _slot: usize) {}
}
impl LayerData for PersistentCompactIntMatrix {
impl LayerData for PersistentIntMatrix {
type Item = Box<[u32]>;
fn open(layer_dir: &Path) -> OKIResult<Self> {
PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)
PersistentIntMatrix::open(layer_dir).map_err(OKIError::Io)
}
fn read(&self, slot: usize) -> Box<[u32]> {
self.row(slot)
@@ -112,7 +112,7 @@ pub trait HasLayerContent {
const CONTENT: LayerContent;
}
impl HasLayerContent for PersistentCompactIntMatrix {
impl HasLayerContent for PersistentIntMatrix {
const CONTENT: LayerContent = LayerContent::Count;
}
@@ -135,9 +135,9 @@ pub trait HasStorageKind {
fn storage_kind(&self) -> obicompactvec::StorageKind;
}
impl HasStorageKind for PersistentCompactIntMatrix {
impl HasStorageKind for PersistentIntMatrix {
fn storage_kind(&self) -> obicompactvec::StorageKind {
PersistentCompactIntMatrix::storage_kind(self)
PersistentIntMatrix::storage_kind(self)
}
}
@@ -342,7 +342,7 @@ impl TypedLayer<()> {
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
impl TypedLayer<PersistentCompactIntMatrix> {
impl TypedLayer<PersistentIntMatrix> {
pub fn build(
out_dir: &Path,
block_bits: u8,
@@ -377,12 +377,12 @@ impl TypedLayer<PersistentCompactIntMatrix> {
// ── Mode 2 — count matrix column append ──────────────────────────────────────
impl TypedLayer<PersistentCompactIntMatrix> {
impl TypedLayer<PersistentIntMatrix> {
pub fn append_genome_column(
layer_dir: &Path,
value_of: impl Fn(usize) -> u32,
) -> OKIResult<()> {
PersistentCompactIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of)
PersistentIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of)
.map_err(OKIError::Io)
}
@@ -418,7 +418,10 @@ impl TypedLayer<PersistentCompactIntMatrix> {
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
/// delegates to `PersistentCompactIntMatrix::nonzero_iter`.
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
self.data.nonzero_iter(slots)
}
}
@@ -477,7 +480,10 @@ impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix + obicompactvec::ColumnWeig
impl TypedLayer<PersistentBitMatrix> {
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
/// delegates to `PersistentBitMatrix::nonzero_iter`.
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
self.data.nonzero_iter(slots)
}
src/obikindexer/src/extensions/private_builder.rs
+9 -10
View File
@@ -24,14 +24,14 @@ use std::fs;
use std::io;
use std::path::Path;
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::prelude::*;
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{KmerLayer, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult};
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
@@ -139,7 +139,9 @@ impl PrivateBuilder for KmerIndex {
mode: &IndexMode,
block_bits: u8,
) -> Result<usize, SKError> {
let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?;
let layer0 = self
.layer0(i)
.map_err(|e| io::Error::other(e.to_string()))?;
let layer0_dir = layer0.dir();
let dedup_path = layer0.dereplicated_superkmers_path();
if !dedup_path.exists() {
@@ -193,15 +195,12 @@ impl PrivateBuilder for KmerIndex {
let n_kmers = if with_counts {
let n = write_graph_as_unitigs(g, layer0_dir)
.map_err(|e| io::Error::other(e.to_string()))?;
TypedLayer::<PersistentCompactIntMatrix>::build(
layer0_dir,
block_bits,
mode,
|kmer| match (&mphf1_opt, &counts1_opt) {
TypedLayer::<PersistentIntMatrix>::build(layer0_dir, block_bits, mode, |kmer| {
match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1,
},
)
}
})
.map_err(|e| io::Error::other(e.to_string()))?;
n
} else {
src/obikmerge/src/partition_merge.rs
+16 -8
View File
@@ -34,7 +34,7 @@ fn presence(mode: MergeMode) -> bool {
mod matrix_builder_tests {
use tempfile::tempdir;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
use super::{ColBuilder, MatrixBuilder};
@@ -90,7 +90,7 @@ mod matrix_builder_tests {
col.close().unwrap();
mb.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 2);
assert_eq!(&*m.row(0), &[0u32, 7]);
assert_eq!(&*m.row(1), &[0u32, 42]);
@@ -422,8 +422,13 @@ pub(crate) fn merge_partition(
move |data: Pass2Data,
push: &PipelineSender<Result<Pass2Data, PipelineError>>,
delta: &PipelineSender<isize>| {
if let Pass2Data::SrcLayer((col_offset, src_n, unitigs_path, src_layer, _guard)) =
data
if let Pass2Data::SrcLayer((
col_offset,
src_n,
unitigs_path,
src_layer,
_guard,
)) = data
{
// _guard dropped at end of block, releasing the slot.
// `src_layer` (MPHF + matrix) was already opened up
@@ -455,8 +460,10 @@ pub(crate) fn merge_partition(
}
}
if !batch.is_empty() {
push.send(Ok(Pass2Data::RawBatch((col_offset, src_n, src_layer, batch))))
.ok();
push.send(Ok(Pass2Data::RawBatch((
col_offset, src_n, src_layer, batch,
))))
.ok();
count += 1;
}
delta.send(count - 1).ok();
@@ -480,8 +487,9 @@ pub(crate) fn merge_partition(
// per kmer — matches the underlying matrix's own
// column-major layout.
let slots = src_layer.hash_batch(&kmers);
let mut cols: Vec<Vec<u32>> =
(0..src_n).map(|_| Vec::with_capacity(kmers.len())).collect();
let mut cols: Vec<Vec<u32>> = (0..src_n)
.map(|_| Vec::with_capacity(kmers.len()))
.collect();
src_layer.fill_sub_matrix(&slots, &mut cols);
// Membership against dst, grouped by dst layer
src/obikphylo/src/matrix_store.rs
+4 -4
View File
@@ -1,15 +1,15 @@
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
use obikidxcache::LayeredStore;
use obikindex::OKIResult;
use obikindex::layer::open_data;
use obikidxcache::LayeredStore;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikindex::load_meta;
use obikindex::KmerIndex;
use obikindex::load_meta;
impl KmerIndex {
/// Open all count matrices for partition `part`, one per layer.
/// Layers without a `counts/` directory are skipped.
pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentCompactIntMatrix>> {
pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentIntMatrix>> {
let index_dir = self.index_dir(part);
if !index_dir.exists() {
return Ok(LayeredStore::new(vec![]));
src/obikselect/src/select_layer.rs
+23 -12
View File
@@ -11,8 +11,8 @@ use std::io;
use std::path::Path;
use obicompactvec::{
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix,
PersistentCompactIntMatrix, TempBitVec, TempCompactIntVec,
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix, PersistentIntMatrix, TempBitVec,
TempCompactIntVec,
};
use obikindex::layer::{KmerLayer, LayerContent};
use obikindex::{KmerIndex, OKIError, OKIResult};
@@ -39,13 +39,15 @@ impl AggOp {
/// `--aggregate-op`.
pub fn parse(s: &str) -> Result<Self, String> {
match s.to_lowercase().as_str() {
"any" => Ok(AggOp::Any),
"all" => Ok(AggOp::All),
"any" => Ok(AggOp::Any),
"all" => Ok(AggOp::All),
"none" => Ok(AggOp::None),
"sum" => Ok(AggOp::Sum),
"min" => Ok(AggOp::Min),
"max" => Ok(AggOp::Max),
other => Err(format!("unknown aggregation operator: {other}; valid: any, all, none, sum, min, max")),
"sum" => Ok(AggOp::Sum),
"min" => Ok(AggOp::Min),
"max" => Ok(AggOp::Max),
other => Err(format!(
"unknown aggregation operator: {other}; valid: any, all, none, sum, min, max"
)),
}
}
@@ -75,7 +77,11 @@ enum AggResult {
Int(TempCompactIntVec),
}
fn compute_group(mat: &dyn MatrixGroupOps, spec: &OutputCol, threshold: u32) -> io::Result<AggResult> {
fn compute_group(
mat: &dyn MatrixGroupOps,
spec: &OutputCol,
threshold: u32,
) -> io::Result<AggResult> {
let g = ColGroup::new(spec.label.clone(), spec.indices.clone());
Ok(match spec.op {
AggOp::Any => AggResult::Bit(mat.partial_group_any(&g, threshold)?),
@@ -145,18 +151,23 @@ pub(crate) fn select_partition(
let group_mat: Box<dyn MatrixGroupOps> = match src_layer.content() {
LayerContent::Count => {
Box::new(PersistentCompactIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
Box::new(PersistentIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
}
LayerContent::Presence => {
Box::new(PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
}
};
let data_subdir = if output_presence { "presence" } else { "counts" };
let data_subdir = if output_presence {
"presence"
} else {
"counts"
};
let data_dir = dst_layer_dir.join(data_subdir);
fs::create_dir_all(&data_dir).map_err(OKIError::Io)?;
let mut builder = MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?;
let mut builder =
MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?;
for spec in specs {
let r = compute_group(group_mat.as_ref(), spec, threshold).map_err(OKIError::Io)?;
add_result(&mut builder, r).map_err(OKIError::Io)?;