OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Rename PersistentCompactIntMatrix to PersistentIntMatrix

Browse Source 
Update all matrix type references, imports, and instantiations across
the codebase to use the new PersistentIntMatrix name. The change also
includes standardizing code formatting, such as multi-line statements
and import ordering, without altering any behavioral logic or public
API contracts.
This commit is contained in:
Eric Coissac
2026-08-28 19:59:59 +02:00
parent 6bdc9354d3
commit 4b6005962e
14 changed files with 779 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/obicompactvec/examples/compare_sparse_dense_count.rs
+29 -10
View File
@@ -13,20 +13,30 @@ use std::error::Error;
use std::path::Path;
use std::time::Instant;
use obicompactvec::{PersistentCompactIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder};
use obicompactvec::{
PersistentIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
};
fn main() -> Result<(), Box<dyn Error>> {
let layer_dir = std::env::args().nth(1).expect("usage: compare_sparse_dense_count <layer_dir>");
let layer_dir = std::env::args()
.nth(1)
.expect("usage: compare_sparse_dense_count <layer_dir>");
let layer_dir = Path::new(&layer_dir);
let t0 = Instant::now();
let dense = PersistentCompactIntMatrix::open(layer_dir)?;
println!("dense ouverte en {:?} ({} lignes x {} colonnes)", t0.elapsed(), dense.n(), dense.n_cols());
let dense = PersistentIntMatrix::open(layer_dir)?;
println!(
"dense ouverte en {:?} ({} lignes x {} colonnes)",
t0.elapsed(),
dense.n(),
dense.n_cols()
);
let out_dir = tempfile::tempdir()?;
let t0 = Instant::now();
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?.finish()?;
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?
.finish()?;
println!("build_from_dense: {:?}", t0.elapsed());
compare(&dense, &sparse)?;
@@ -36,7 +46,10 @@ fn main() -> Result<(), Box<dyn Error>> {
Ok(())
}
fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactIntMatrix) -> Result<(), Box<dyn Error>> {
fn compare(
dense: &PersistentIntMatrix,
sparse: &PersistentSparseCompactIntMatrix,
) -> Result<(), Box<dyn Error>> {
let n = dense.n();
let n_cols = dense.n_cols();
assert_eq!(n, sparse.n(), "n mismatch");
@@ -63,7 +76,10 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI
}
}
}
println!("comparaison case-a-case: {:?} ({total_cells} cellules)", t0.elapsed());
println!(
"comparaison case-a-case: {:?} ({total_cells} cellules)",
t0.elapsed()
);
if let Some((slot, col, d, s)) = first_mismatch {
eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}");
@@ -78,7 +94,7 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI
/// for what's actually stored) — sparsity, singleton-row fraction, and the
/// value-magnitude buckets this crate's overflow encoding is tuned around
/// (< 127, < 255, >= 255).
fn content_stats(dense: &PersistentCompactIntMatrix) {
fn content_stats(dense: &PersistentIntMatrix) {
let n = dense.n();
let n_cols = dense.n_cols();
let total_cells = n as u64 * n_cols as u64;
@@ -128,7 +144,10 @@ fn content_stats(dense: &PersistentCompactIntMatrix) {
100.0 * singleton_rows as f64 / n as f64
);
if nonzero_cells > 0 {
println!("valeur moyenne (non-nulles): {:.2}", sum as f64 / nonzero_cells as f64);
println!(
"valeur moyenne (non-nulles): {:.2}",
sum as f64 / nonzero_cells as f64
);
println!("valeur max: {max_value}");
println!(
"valeurs < 127: {under_127} ({:.3}% des non-nulles)",
@@ -162,7 +181,7 @@ fn dir_size(dir: &Path) -> std::io::Result<u64> {
fn compaction_stats(
dense_layer_dir: &Path,
sparse_dir: &Path,
dense: &PersistentCompactIntMatrix,
dense: &PersistentIntMatrix,
_sparse: &PersistentSparseCompactIntMatrix,
) -> Result<(), Box<dyn Error>> {
let dense_size = dir_size(&dense_layer_dir.join("counts"))?;
src/obicompactvec/src/intmatrix.rs
+288 -111
View File
@@ -5,17 +5,19 @@ use std::path::{Path, PathBuf};
use memmap2::Mmap;
use ndarray::{Array1, Array2};
use crate::bitmatrix::{fill_row_generic, par_col_reduce, pairwise2_matrix, pairwise_matrix, row_generic};
use crate::bitmatrix::{
fill_row_generic, pairwise_matrix, pairwise2_matrix, par_col_reduce, row_generic,
};
use crate::builder::PersistentCompactIntVecBuilder;
use crate::colgroup::{chunked_presence_count, ColGroup, MatrixGroupOps};
use crate::colgroup::{ColGroup, MatrixGroupOps, chunked_presence_count};
use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE};
use crate::meta::MatrixMeta;
use crate::reader::PersistentCompactIntVec;
use crate::sparse_intmatrix::{is_present as sparse_is_present, PersistentSparseCompactIntMatrix};
use crate::sparse_intmatrix::{PersistentSparseCompactIntMatrix, is_present as sparse_is_present};
use crate::storage_kind::StorageKind;
use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
use crate::views::{nonzero_triples, IntSliceView};
use crate::views::{IntSliceView, nonzero_triples};
pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf {
dir.join(format!("col_{col:06}.pciv"))
@@ -25,7 +27,7 @@ pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf {
pub struct ColumnarCompactIntMatrix {
cols: Vec<PersistentCompactIntVec>,
n: usize,
n: usize,
}
impl ColumnarCompactIntMatrix {
@@ -38,11 +40,17 @@ impl ColumnarCompactIntMatrix {
}
#[inline]
pub(crate) fn n(&self) -> usize { self.n }
pub(crate) fn n(&self) -> usize {
self.n
}
#[inline]
pub(crate) fn n_cols(&self) -> usize { self.cols.len() }
pub(crate) fn n_cols(&self) -> usize {
self.cols.len()
}
#[inline]
pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec { &self.cols[c] }
pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec {
&self.cols[c]
}
#[inline]
pub(crate) fn row(&self, slot: usize) -> Box<[u32]> {
@@ -63,34 +71,64 @@ impl ColumnarCompactIntMatrix {
}
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_bray_dist(self.col(j)))
pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_bray_dist(self.col(j))
})
}
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_euclidean_dist(self.col(j)))
pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_euclidean_dist(self.col(j))
})
}
pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
pairwise2_matrix(self.n_cols(), |i, j| self.col(i).partial_threshold_jaccard_dist(self.col(j), threshold))
pub(crate) fn partial_threshold_jaccard_dist_matrix(
&self,
threshold: u32,
) -> (Array2<u64>, Array2<u64>) {
pairwise2_matrix(self.n_cols(), |i, j| {
self.col(i)
.partial_threshold_jaccard_dist(self.col(j), threshold)
})
}
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_relfreq_bray_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
self.col(i).partial_relfreq_bray_dist(
self.col(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pub(crate) fn partial_relfreq_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_relfreq_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
self.col(i).partial_relfreq_euclidean_dist(
self.col(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pub(crate) fn partial_hellinger_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_hellinger_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
self.col(i).partial_hellinger_euclidean_dist(
self.col(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
pub(crate) fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> {
let mut meta = MatrixMeta::load(dir)?;
let mut b = PersistentCompactIntVecBuilder::new(meta.n, &col_path(dir, meta.n_cols))?;
for slot in 0..meta.n { b.set(slot, value_of(slot)); }
for slot in 0..meta.n {
b.set(slot, value_of(slot));
}
b.close()?;
meta.n_cols += 1;
meta.save(dir)
@@ -99,19 +137,19 @@ impl ColumnarCompactIntMatrix {
// ── PackedCompactIntMatrix ────────────────────────────────────────────────────
const PCMX_MAGIC: [u8; 4] = *b"PCMX";
const PCMX_HEADER: usize = 24; // magic(4) + pad(4) + n_rows(8) + n_cols(8)
const PCMX_MAGIC: [u8; 4] = *b"PCMX";
const PCMX_HEADER: usize = 24; // magic(4) + pad(4) + n_rows(8) + n_cols(8)
struct ColInfo {
primary_start: usize,
data_offset: usize,
n_overflow: usize,
data_offset: usize,
n_overflow: usize,
}
pub struct PackedCompactIntMatrix {
mmap: Mmap,
n_rows: usize,
n_cols: usize,
mmap: Mmap,
n_rows: usize,
n_cols: usize,
columns: Vec<ColInfo>,
}
@@ -119,43 +157,65 @@ impl PackedCompactIntMatrix {
pub(crate) fn open(path: &Path) -> io::Result<Self> {
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
if mmap.len() < PCMX_HEADER {
return Err(io::Error::new(io::ErrorKind::InvalidData, "PCMX file too short"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"PCMX file too short",
));
}
if &mmap[0..4] != &PCMX_MAGIC {
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PCMX magic"));
}
let n_rows = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
let n_rows = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
let n_cols = u64::from_le_bytes(mmap[16..24].try_into().unwrap()) as usize;
let mut columns = Vec::with_capacity(n_cols);
for c in 0..n_cols {
let off_pos = PCMX_HEADER + c * 8;
let col_base = u64::from_le_bytes(mmap[off_pos..off_pos+8].try_into().unwrap()) as usize;
let n_ov = u64::from_le_bytes(mmap[col_base+16..col_base+24].try_into().unwrap()) as usize;
let n_pciv = u64::from_le_bytes(mmap[col_base+8..col_base+16].try_into().unwrap()) as usize;
let off_pos = PCMX_HEADER + c * 8;
let col_base =
u64::from_le_bytes(mmap[off_pos..off_pos + 8].try_into().unwrap()) as usize;
let n_ov =
u64::from_le_bytes(mmap[col_base + 16..col_base + 24].try_into().unwrap()) as usize;
let n_pciv =
u64::from_le_bytes(mmap[col_base + 8..col_base + 16].try_into().unwrap()) as usize;
let primary_start = col_base + HEADER_SIZE;
let data_offset = primary_start + n_pciv;
columns.push(ColInfo { primary_start, data_offset, n_overflow: n_ov });
let data_offset = primary_start + n_pciv;
columns.push(ColInfo {
primary_start,
data_offset,
n_overflow: n_ov,
});
}
Ok(Self { mmap, n_rows, n_cols, columns })
Ok(Self {
mmap,
n_rows,
n_cols,
columns,
})
}
#[inline]
pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> {
let ci = &self.columns[c];
let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
let overflow_raw = &self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE];
let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
let overflow_raw =
&self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE];
IntSliceView::new(primary, overflow_raw, ci.n_overflow, self.n_rows)
}
pub(crate) fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentCompactIntVecBuilder> {
pub(crate) fn col_persist(
&self,
c: usize,
path: &Path,
) -> io::Result<PersistentCompactIntVecBuilder> {
let view = self.col_view(c);
let overflow: std::collections::HashMap<usize, u32> = view.overflow_entries().collect();
PersistentCompactIntVecBuilder::from_raw_primary(view.primary_bytes(), overflow, path)
}
#[inline]
pub(crate) fn get(&self, col: usize, slot: usize) -> u32 { self.col_view(col).get(slot) }
pub(crate) fn get(&self, col: usize, slot: usize) -> u32 {
self.col_view(col).get(slot)
}
#[inline]
pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
@@ -180,27 +240,55 @@ impl PackedCompactIntMatrix {
// via `PersistentCompactIntVec`. No formula is re-derived here.
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_bray_dist(self.col_view(j)))
pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_bray_dist(self.col_view(j))
})
}
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_euclidean_dist(self.col_view(j)))
pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_euclidean_dist(self.col_view(j))
})
}
pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, t: u32) -> (Array2<u64>, Array2<u64>) {
pairwise2_matrix(self.n_cols, |i, j| self.col_view(i).partial_threshold_jaccard_dist(self.col_view(j), t))
pub(crate) fn partial_threshold_jaccard_dist_matrix(
&self,
t: u32,
) -> (Array2<u64>, Array2<u64>) {
pairwise2_matrix(self.n_cols, |i, j| {
self.col_view(i)
.partial_threshold_jaccard_dist(self.col_view(j), t)
})
}
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_relfreq_bray_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
self.col_view(i).partial_relfreq_bray_dist(
self.col_view(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pub(crate) fn partial_relfreq_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_relfreq_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
self.col_view(i).partial_relfreq_euclidean_dist(
self.col_view(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
pub(crate) fn partial_hellinger_euclidean_dist_matrix(
&self,
col_sums: &Array1<u64>,
) -> Array2<f64> {
pairwise_matrix(self.n_cols, |i, j| {
self.col_view(i).partial_hellinger_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
self.col_view(i).partial_hellinger_euclidean_dist(
self.col_view(j),
col_sums[i] as f64,
col_sums[j] as f64,
)
})
}
}
@@ -238,7 +326,9 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
// the columnar files are newer and must be (re-)packed, overwriting the
// stale one — never silently discarded as "leftover cleanup".
if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
for c in 0..meta.n_cols {
let _ = fs::remove_file(col_path(dir, c));
}
let _ = fs::remove_file(dir.join("meta.json"));
return Ok(());
}
@@ -250,32 +340,41 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
let header_size = (PCMX_HEADER + n_cols * 8) as u64;
let mut col_offset = header_size;
let mut offsets = Vec::with_capacity(n_cols);
for &size in &col_sizes { offsets.push(col_offset); col_offset += size; }
for &size in &col_sizes {
offsets.push(col_offset);
col_offset += size;
}
let tmp_path = dir.join("matrix.pcmx.tmp");
let mut out = BufWriter::new(File::create(&tmp_path)?);
out.write_all(&PCMX_MAGIC)?;
out.write_all(&[0u8; 4])?;
out.write_all(&(meta.n as u64).to_le_bytes())?;
out.write_all(&(n_cols as u64).to_le_bytes())?;
for &off in &offsets { out.write_all(&off.to_le_bytes())?; }
for c in 0..n_cols { io::copy(&mut File::open(col_path(dir, c))?, &mut out)?; }
for &off in &offsets {
out.write_all(&off.to_le_bytes())?;
}
for c in 0..n_cols {
io::copy(&mut File::open(col_path(dir, c))?, &mut out)?;
}
out.flush()?;
drop(out);
fs::rename(&tmp_path, &packed_path)?;
for c in 0..n_cols { fs::remove_file(col_path(dir, c))?; }
for c in 0..n_cols {
fs::remove_file(col_path(dir, c))?;
}
fs::remove_file(dir.join("meta.json"))?;
Ok(())
}
// ── PersistentCompactIntMatrix — public enum ──────────────────────────────────
pub enum PersistentCompactIntMatrix {
pub enum PersistentIntMatrix {
Columnar(ColumnarCompactIntMatrix),
Packed(PackedCompactIntMatrix),
Sparse(PersistentSparseCompactIntMatrix),
}
impl PersistentCompactIntMatrix {
impl PersistentIntMatrix {
/// Checks (in order): `counts/matrix.pcmx` → Packed; `counts/meta.json`
/// → Columnar; `counts/singleton_values.pciv` → Sparse. Mirrors
/// `PersistentBitMatrix::open`'s own Packed → Columnar → Sparse
@@ -283,17 +382,24 @@ impl PersistentCompactIntMatrix {
pub fn open(layer_dir: &Path) -> io::Result<Self> {
let counts_dir = layer_dir.join("counts");
if counts_dir.join("matrix.pcmx").exists() {
return Ok(Self::Packed(PackedCompactIntMatrix::open(&counts_dir.join("matrix.pcmx"))?));
return Ok(Self::Packed(PackedCompactIntMatrix::open(
&counts_dir.join("matrix.pcmx"),
)?));
}
if MatrixMeta::load(&counts_dir).is_ok() {
return Ok(Self::Columnar(ColumnarCompactIntMatrix::open(&counts_dir)?));
}
if sparse_is_present(&counts_dir) {
return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open(&counts_dir)?));
return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open(
&counts_dir,
)?));
}
Err(io::Error::new(
io::ErrorKind::NotFound,
format!("no count matrix found in {} — run 'obikmer upgrade'", layer_dir.display()),
format!(
"no count matrix found in {} — run 'obikmer upgrade'",
layer_dir.display()
),
))
}
@@ -303,8 +409,8 @@ impl PersistentCompactIntMatrix {
pub fn storage_kind(&self) -> StorageKind {
match self {
Self::Columnar(_) => StorageKind::Columnar,
Self::Packed(_) => StorageKind::Packed,
Self::Sparse(_) => StorageKind::Sparse,
Self::Packed(_) => StorageKind::Packed,
Self::Sparse(_) => StorageKind::Sparse,
}
}
@@ -330,11 +436,19 @@ impl PersistentCompactIntMatrix {
#[inline]
pub fn n(&self) -> usize {
match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows, Self::Sparse(m) => m.n() }
match self {
Self::Columnar(m) => m.n(),
Self::Packed(m) => m.n_rows,
Self::Sparse(m) => m.n(),
}
}
#[inline]
pub fn n_cols(&self) -> usize {
match self { Self::Columnar(m) => m.n_cols(), Self::Packed(m) => m.n_cols, Self::Sparse(m) => m.n_cols() }
match self {
Self::Columnar(m) => m.n_cols(),
Self::Packed(m) => m.n_cols,
Self::Sparse(m) => m.n_cols(),
}
}
#[inline]
@@ -349,30 +463,38 @@ impl PersistentCompactIntMatrix {
pub fn col_view(&self, c: usize) -> IntSliceView<'_> {
match self {
Self::Columnar(m) => m.col(c).view(),
Self::Packed(m) => m.col_view(c),
Self::Sparse(_) => panic!("col_view() not available on Sparse PersistentCompactIntMatrix"),
Self::Packed(m) => m.col_view(c),
Self::Sparse(_) => {
panic!("col_view() not available on Sparse PersistentCompactIntMatrix")
}
}
}
pub fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentCompactIntVecBuilder> {
match self {
Self::Columnar(m) => PersistentCompactIntVecBuilder::build_from(m.col(c), path),
Self::Packed(m) => m.col_persist(c, path),
Self::Sparse(_) => Err(io::Error::new(io::ErrorKind::Unsupported,
"col_persist not available on Sparse PersistentCompactIntMatrix")),
Self::Packed(m) => m.col_persist(c, path),
Self::Sparse(_) => Err(io::Error::new(
io::ErrorKind::Unsupported,
"col_persist not available on Sparse PersistentCompactIntMatrix",
)),
}
}
#[inline]
pub fn row(&self, slot: usize) -> Box<[u32]> {
match self { Self::Columnar(m) => m.row(slot), Self::Packed(m) => m.row(slot), Self::Sparse(m) => m.row(slot) }
match self {
Self::Columnar(m) => m.row(slot),
Self::Packed(m) => m.row(slot),
Self::Sparse(m) => m.row(slot),
}
}
#[inline]
pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
match self {
Self::Columnar(m) => m.fill_row(slot, buf),
Self::Packed(m) => m.fill_row(slot, buf),
Self::Sparse(m) => m.fill_row(slot, buf),
Self::Packed(m) => m.fill_row(slot, buf),
Self::Sparse(m) => m.fill_row(slot, buf),
}
}
@@ -418,73 +540,86 @@ impl PersistentCompactIntMatrix {
/// `Columnar`/`Packed` reuse the shared sorted-slot batching in
/// `views::nonzero_triples`. Boxed, not `impl Iterator`, because the
/// match arms are genuinely different concrete types.
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
match self {
Self::Sparse(m) => Box::new(m.nonzero_iter(slots)),
Self::Columnar(_) | Self::Packed(_) => {
Box::new(nonzero_triples(slots, self.n_cols(), |c| self.col_view(c), false))
}
Self::Columnar(_) | Self::Packed(_) => Box::new(nonzero_triples(
slots,
self.n_cols(),
|c| self.col_view(c),
false,
)),
}
}
#[inline]
pub fn sum(&self) -> Array1<u64> {
match self { Self::Columnar(m) => m.sum(), Self::Packed(m) => m.sum(), Self::Sparse(m) => m.sum() }
match self {
Self::Columnar(m) => m.sum(),
Self::Packed(m) => m.sum(),
Self::Sparse(m) => m.sum(),
}
}
#[inline]
pub fn count_nonzero(&self) -> Array1<u64> {
match self {
Self::Columnar(m) => m.count_nonzero(),
Self::Packed(m) => m.count_nonzero(),
Self::Sparse(m) => m.count_nonzero(),
Self::Packed(m) => m.count_nonzero(),
Self::Sparse(m) => m.count_nonzero(),
}
}
#[inline]
pub fn partial_bray_dist_matrix(&self) -> Array2<u64> {
match self {
Self::Columnar(m) => m.partial_bray_dist_matrix(),
Self::Packed(m) => m.partial_bray_dist_matrix(),
Self::Sparse(m) => CountPartials::partial_bray(m),
Self::Packed(m) => m.partial_bray_dist_matrix(),
Self::Sparse(m) => CountPartials::partial_bray(m),
}
}
#[inline]
pub fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
match self {
Self::Columnar(m) => m.partial_euclidean_dist_matrix(),
Self::Packed(m) => m.partial_euclidean_dist_matrix(),
Self::Sparse(m) => CountPartials::partial_euclidean(m),
Self::Packed(m) => m.partial_euclidean_dist_matrix(),
Self::Sparse(m) => CountPartials::partial_euclidean(m),
}
}
#[inline]
pub fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
pub fn partial_threshold_jaccard_dist_matrix(
&self,
threshold: u32,
) -> (Array2<u64>, Array2<u64>) {
match self {
Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
Self::Sparse(m) => CountPartials::partial_threshold_jaccard(m, threshold),
Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
Self::Sparse(m) => CountPartials::partial_threshold_jaccard(m, threshold),
}
}
#[inline]
pub fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
match self {
Self::Columnar(m) => m.partial_relfreq_bray_dist_matrix(col_sums),
Self::Packed(m) => m.partial_relfreq_bray_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_relfreq_bray(m, col_sums),
Self::Packed(m) => m.partial_relfreq_bray_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_relfreq_bray(m, col_sums),
}
}
#[inline]
pub fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
match self {
Self::Columnar(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums),
Self::Packed(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_relfreq_euclidean(m, col_sums),
Self::Packed(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_relfreq_euclidean(m, col_sums),
}
}
#[inline]
pub fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
match self {
Self::Columnar(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums),
Self::Packed(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_hellinger(m, col_sums),
Self::Packed(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums),
Self::Sparse(m) => CountPartials::partial_hellinger(m, col_sums),
}
}
#[inline]
@@ -497,40 +632,60 @@ impl PersistentCompactIntMatrix {
use crate::traits::{ColumnWeights, CountPartials};
impl ColumnWeights for PersistentCompactIntMatrix {
impl ColumnWeights for PersistentIntMatrix {
#[inline]
fn col_weights(&self) -> Array1<u64> { self.sum() }
fn col_weights(&self) -> Array1<u64> {
self.sum()
}
#[inline]
fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() }
fn partial_kmer_counts(&self) -> Array1<u64> {
self.count_nonzero()
}
}
impl CountPartials for PersistentCompactIntMatrix {
impl CountPartials for PersistentIntMatrix {
#[inline]
fn partial_bray(&self) -> Array2<u64> { self.partial_bray_dist_matrix() }
fn partial_bray(&self) -> Array2<u64> {
self.partial_bray_dist_matrix()
}
#[inline]
fn partial_euclidean(&self) -> Array2<f64> { self.partial_euclidean_dist_matrix() }
fn partial_euclidean(&self) -> Array2<f64> {
self.partial_euclidean_dist_matrix()
}
#[inline]
fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) { self.partial_threshold_jaccard_dist_matrix(t) }
fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) {
self.partial_threshold_jaccard_dist_matrix(t)
}
#[inline]
fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_bray_dist_matrix(g) }
fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> {
self.partial_relfreq_bray_dist_matrix(g)
}
#[inline]
fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_euclidean_dist_matrix(g) }
fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> {
self.partial_relfreq_euclidean_dist_matrix(g)
}
#[inline]
fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_hellinger_euclidean_dist_matrix(g) }
fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> {
self.partial_hellinger_euclidean_dist_matrix(g)
}
}
// ── Builder ───────────────────────────────────────────────────────────────────
pub struct PersistentCompactIntMatrixBuilder {
dir: PathBuf,
n: usize,
dir: PathBuf,
n: usize,
n_cols: usize,
}
impl PersistentCompactIntMatrixBuilder {
pub fn new(n: usize, dir: &Path) -> io::Result<Self> {
fs::create_dir_all(dir)?;
Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
Ok(Self {
dir: dir.to_path_buf(),
n,
n_cols: 0,
})
}
/// Resume appending columns to a Columnar matrix directory already
@@ -541,13 +696,21 @@ impl PersistentCompactIntMatrixBuilder {
/// on-disk column naming or `MatrixMeta` themselves.
pub fn resume(dir: &Path) -> io::Result<Self> {
let meta = MatrixMeta::load(dir)?;
Ok(Self { dir: dir.to_path_buf(), n: meta.n, n_cols: meta.n_cols })
Ok(Self {
dir: dir.to_path_buf(),
n: meta.n,
n_cols: meta.n_cols,
})
}
#[inline]
pub fn n(&self) -> usize { self.n }
pub fn n(&self) -> usize {
self.n
}
#[inline]
pub fn n_cols(&self) -> usize { self.n_cols }
pub fn n_cols(&self) -> usize {
self.n_cols
}
pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> {
let path = col_path(&self.dir, self.n_cols);
self.n_cols += 1;
@@ -569,14 +732,22 @@ impl PersistentCompactIntMatrixBuilder {
}
pub fn close(self) -> io::Result<()> {
MatrixMeta { n: self.n, n_cols: self.n_cols }.save(&self.dir)
MatrixMeta {
n: self.n,
n_cols: self.n_cols,
}
.save(&self.dir)
}
}
// ── MatrixGroupOps ────────────────────────────────────────────────────────────
impl MatrixGroupOps for PersistentCompactIntMatrix {
fn partial_group_presence_count(&self, g: &ColGroup, threshold: u32) -> io::Result<TempCompactIntVec> {
impl MatrixGroupOps for PersistentIntMatrix {
fn partial_group_presence_count(
&self,
g: &ColGroup,
threshold: u32,
) -> io::Result<TempCompactIntVec> {
chunked_presence_count(self.n(), &g.indices, |b, c| {
b.inc_predicate_fast(self.col_view(c), |v| v >= threshold)
})
@@ -585,7 +756,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
let n = self.n();
let mut result = TempCompactIntVecBuilder::new(n)?;
for &c in &g.indices { result.add(self.col_view(c)); }
for &c in &g.indices {
result.add(self.col_view(c));
}
result.freeze()
}
@@ -603,7 +776,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
let mut result = TempCompactIntVecBuilder::new(n)?;
if let Some((&first, rest)) = g.indices.split_first() {
result.add(self.col_view(first));
for &c in rest { result.min(self.col_view(c)); }
for &c in rest {
result.min(self.col_view(c));
}
}
result.freeze()
}
@@ -611,7 +786,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
let n = self.n();
let mut result = TempCompactIntVecBuilder::new(n)?;
for &c in &g.indices { result.max(self.col_view(c)); }
for &c in &g.indices {
result.max(self.col_view(c));
}
result.freeze()
}
}
src/obicompactvec/src/lib.rs
+20 -11
View File
@@ -1,41 +1,50 @@
mod bitvec;
mod bitmatrix;
mod bitvec;
mod builder;
mod colgroup;
mod eliasfano;
mod fixedintvec;
mod format;
mod rankselect;
mod intmatrix;
mod layer_meta;
mod matrix_builder;
mod meta;
mod mmap_file;
mod rankselect;
mod reader;
mod sparse_intmatrix;
mod storage_kind;
mod tempbitvec;
mod tempintvec;
mod views;
pub mod traits;
mod views;
pub use bitmatrix::{
PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix,
PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix,
};
pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
pub use eliasfano::{EliasFano, EliasFanoBuilder};
pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix};
pub use builder::PersistentCompactIntVecBuilder;
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
pub use eliasfano::{EliasFano, EliasFanoBuilder};
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
pub use intmatrix::{
PersistentCompactIntMatrixBuilder, PersistentIntMatrix, pack_compact_int_matrix,
};
pub use layer_meta::LayerMeta;
pub use matrix_builder::{ColBuilder, MatrixBuilder};
pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter};
pub use sparse_intmatrix::{PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder, pack_sparse_compact_int_matrix};
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
pub use reader::{Iter as CompactIntVecIter, PersistentCompactIntVec};
pub use sparse_intmatrix::{
PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
pack_sparse_compact_int_matrix,
};
pub use storage_kind::StorageKind;
pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials};
pub use views::{BitSliceView, BitSliceIter, IntSliceView, IntSliceViewIter};
pub use views::{BitSliceIter, BitSliceView, IntSliceView, IntSliceViewIter};
#[cfg(test)]
#[path = "tests/mod.rs"]
src/obicompactvec/src/sparse_intmatrix.rs
+101 -41
View File
@@ -35,11 +35,17 @@ use crate::builder::PersistentCompactIntVecBuilder;
use crate::eliasfano::{EliasFano, EliasFanoBuilder};
use crate::reader::PersistentCompactIntVec;
use crate::traits::{BitPartials, ColumnWeights, CountPartials};
use crate::{PersistentCompactIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
use crate::{PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
fn singleton_values_path(dir: &Path) -> PathBuf { dir.join("singleton_values.pciv") }
fn multi_values_path(dir: &Path) -> PathBuf { dir.join("multi_values.pciv") }
fn multi_offsets_base(dir: &Path) -> PathBuf { dir.join("multi_offsets") }
fn singleton_values_path(dir: &Path) -> PathBuf {
dir.join("singleton_values.pciv")
}
fn multi_values_path(dir: &Path) -> PathBuf {
dir.join("multi_values.pciv")
}
fn multi_offsets_base(dir: &Path) -> PathBuf {
dir.join("multi_offsets")
}
/// Lightweight disk probe: true iff a sparse count matrix has already been
/// written at `dir` — the marker [`PersistentCompactIntMatrix::open`]/
@@ -73,9 +79,13 @@ impl PersistentSparseCompactIntMatrix {
}
#[inline]
pub fn n(&self) -> usize { self.support.n() }
pub fn n(&self) -> usize {
self.support.n()
}
#[inline]
pub fn n_cols(&self) -> usize { self.support.n_cols() }
pub fn n_cols(&self) -> usize {
self.support.n_cols()
}
/// Calls `f(col, value)` once per non-zero column at `slot`, in
/// ascending column order — the shared decode branch behind every
@@ -104,7 +114,11 @@ impl PersistentSparseCompactIntMatrix {
/// absent).
pub fn get(&self, c: usize, slot: usize) -> u32 {
let mut found = 0u32;
self.for_each_cell_in_row(slot, |g, v| if g == c { found = v; });
self.for_each_cell_in_row(slot, |g, v| {
if g == c {
found = v;
}
});
found
}
@@ -138,27 +152,32 @@ impl PersistentSparseCompactIntMatrix {
/// Yields every `(idx into slots, col, value)` triple, row by row, in
/// `slots` order — mirrors
/// [`PersistentSparseBitMatrix::nonzero_iter`].
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator<Item = (usize, usize, u32)> + 'a {
pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> impl Iterator<Item = (usize, usize, u32)> + 'a {
let mut next_slot = 0usize;
let mut cur_idx = 0usize;
let mut buf: Vec<(usize, u32)> = Vec::new();
let mut buf_pos = 0usize;
std::iter::from_fn(move || loop {
if buf_pos < buf.len() {
let (g, v) = buf[buf_pos];
buf_pos += 1;
return Some((cur_idx, g, v));
std::iter::from_fn(move || {
loop {
if buf_pos < buf.len() {
let (g, v) = buf[buf_pos];
buf_pos += 1;
return Some((cur_idx, g, v));
}
if next_slot >= slots.len() {
return None;
}
cur_idx = next_slot;
let slot = slots[next_slot];
next_slot += 1;
buf.clear();
self.for_each_cell_in_row(slot, |g, v| buf.push((g, v)));
buf_pos = 0;
}
if next_slot >= slots.len() {
return None;
}
cur_idx = next_slot;
let slot = slots[next_slot];
next_slot += 1;
buf.clear();
self.for_each_cell_in_row(slot, |g, v| buf.push((g, v)));
buf_pos = 0;
})
}
@@ -203,7 +222,11 @@ impl PersistentSparseCompactIntMatrix {
fn count_geq(&self, threshold: u32) -> Array1<u64> {
let mut counts = vec![0u64; self.n_cols()];
for slot in 0..self.n() {
self.for_each_cell_in_row(slot, |g, v| if v >= threshold { counts[g] += 1; });
self.for_each_cell_in_row(slot, |g, v| {
if v >= threshold {
counts[g] += 1;
}
});
}
Array1::from(counts)
}
@@ -261,9 +284,13 @@ impl PersistentSparseCompactIntMatrix {
impl ColumnWeights for PersistentSparseCompactIntMatrix {
#[inline]
fn col_weights(&self) -> Array1<u64> { self.sum() }
fn col_weights(&self) -> Array1<u64> {
self.sum()
}
#[inline]
fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() }
fn partial_kmer_counts(&self) -> Array1<u64> {
self.count_nonzero()
}
}
impl CountPartials for PersistentSparseCompactIntMatrix {
@@ -278,7 +305,9 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
fn partial_bray(&self) -> Array2<u64> {
let mut m = self.row_major_pairwise(|_, a, _, b| (a as u64).min(b as u64));
let sum = self.sum();
for i in 0..self.n_cols() { m[[i, i]] = sum[i]; }
for i in 0..self.n_cols() {
m[[i, i]] = sum[i];
}
m
}
@@ -329,7 +358,8 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
// left to patch.
return BitPartials::partial_jaccard(&self.support);
}
let mut inter = self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64);
let mut inter =
self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64);
let mut union = Array2::<u64>::zeros((n, n));
for i in 0..n {
for j in 0..n {
@@ -363,7 +393,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
});
let sum = self.sum();
for i in 0..self.n_cols() {
m[[i, i]] = if global[i] > 0 { sum[i] as f64 / global[i] as f64 } else { 0.0 };
m[[i, i]] = if global[i] > 0 {
sum[i] as f64 / global[i] as f64
} else {
0.0
};
}
m
}
@@ -382,9 +416,21 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
if i != j {
let sa = global[i] as f64;
let sb = global[j] as f64;
let sq_a = if sa > 0.0 { sq[i] as f64 / (sa * sa) } else { 0.0 };
let sq_b = if sb > 0.0 { sq[j] as f64 / (sb * sb) } else { 0.0 };
let cross = if sa > 0.0 && sb > 0.0 { dot[[i, j]] / (sa * sb) } else { 0.0 };
let sq_a = if sa > 0.0 {
sq[i] as f64 / (sa * sa)
} else {
0.0
};
let sq_b = if sb > 0.0 {
sq[j] as f64 / (sb * sb)
} else {
0.0
};
let cross = if sa > 0.0 && sb > 0.0 {
dot[[i, j]] / (sa * sb)
} else {
0.0
};
m[[i, j]] = sq_a + sq_b - 2.0 * cross;
}
}
@@ -409,7 +455,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
let sb = global[j] as f64;
let pa_sum = if sa > 0.0 { sum[i] as f64 / sa } else { 0.0 };
let pb_sum = if sb > 0.0 { sum[j] as f64 / sb } else { 0.0 };
let cross = if sa > 0.0 && sb > 0.0 { sqrt_dot[[i, j]] / (sa * sb).sqrt() } else { 0.0 };
let cross = if sa > 0.0 && sb > 0.0 {
sqrt_dot[[i, j]] / (sa * sb).sqrt()
} else {
0.0
};
m[[i, j]] = pa_sum + pb_sum - 2.0 * cross;
}
}
@@ -461,7 +511,6 @@ impl PersistentSparseCompactIntMatrixBuilder {
self.support.push_row(cols);
}
/// Builds a sparse matrix from an already-built dense
/// [`PersistentCompactIntMatrix`] (`Columnar` or `Packed`) — same
/// batched-transpose shape as
@@ -475,7 +524,7 @@ impl PersistentSparseCompactIntMatrixBuilder {
/// pairs are appended to its own buffer in strictly ascending column
/// order "for free", matching `push_row`'s ascending-`cols` contract
/// without an explicit sort.
pub fn build_from_dense(dense: &PersistentCompactIntMatrix, dir: &Path) -> io::Result<Self> {
pub fn build_from_dense(dense: &PersistentIntMatrix, dir: &Path) -> io::Result<Self> {
let n = dense.n();
let n_cols = dense.n_cols();
let mut builder = Self::new(n, n_cols, dir)?;
@@ -500,23 +549,34 @@ impl PersistentSparseCompactIntMatrixBuilder {
}
pub fn close(self) -> io::Result<()> {
let Self { support, dir, singleton_values, multi_values, multi_row_offsets } = self;
let Self {
support,
dir,
singleton_values,
multi_values,
multi_row_offsets,
} = self;
support.close()?;
let mut sv = PersistentCompactIntVecBuilder::new(singleton_values.len(), &singleton_values_path(&dir))?;
let mut sv = PersistentCompactIntVecBuilder::new(
singleton_values.len(),
&singleton_values_path(&dir),
)?;
for (i, &v) in singleton_values.iter().enumerate() {
sv.set(i, v);
}
sv.close()?;
let mut mv = PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?;
let mut mv =
PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?;
for (i, &v) in multi_values.iter().enumerate() {
mv.set(i, v);
}
mv.close()?;
let universe = multi_values.len() as u64 + 1;
let mut mo = EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?;
let mut mo =
EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?;
for &o in &multi_row_offsets {
mo.push(o);
}
@@ -544,7 +604,7 @@ impl PersistentSparseCompactIntMatrixBuilder {
/// half-written one). Idempotent — a no-op if `singleton_values.pciv`
/// already exists.
pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
use crate::intmatrix::{col_path, ColumnarCompactIntMatrix, PackedCompactIntMatrix};
use crate::intmatrix::{ColumnarCompactIntMatrix, PackedCompactIntMatrix, col_path};
if is_present(dir) {
return Ok(());
@@ -552,12 +612,12 @@ pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
let packed_path = dir.join("matrix.pcmx");
if packed_path.exists() {
let dense = PersistentCompactIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?);
let dense = PersistentIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?);
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
drop(dense);
fs::remove_file(&packed_path)?;
} else {
let dense = PersistentCompactIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?);
let dense = PersistentIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?);
let n_cols = dense.n_cols();
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
drop(dense);
src/obicompactvec/src/tests/colgroup.rs
+33 -20
View File
@@ -1,25 +1,26 @@
use tempfile::tempdir;
use crate::{
ColGroup, MatrixGroupOps,
PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentIntMatrix, PersistentCompactIntMatrixBuilder,
};
use crate::{PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder};
// ── helpers ───────────────────────────────────────────────────────────────────
fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
let n = cols.first().map_or(0, |c| c.len());
let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
for &col in cols {
let mut cb = b.add_col().unwrap();
for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
for (slot, &v) in col.iter().enumerate() {
cb.set(slot, v);
}
cb.close().unwrap();
}
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
(dir, m)
}
@@ -30,7 +31,9 @@ fn make_bit_matrix(cols: &[&[bool]]) -> (tempfile::TempDir, PersistentBitMatrix)
let mut b = PersistentBitMatrixBuilder::new(n, &presence).unwrap();
for &col in cols {
let mut cb = b.add_col().unwrap();
for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
for (slot, &v) in col.iter().enumerate() {
cb.set(slot, v);
}
cb.close().unwrap();
}
b.close().unwrap();
@@ -115,9 +118,13 @@ fn mask_with_zeros_selected_slots() {
// count vec [10, 20, 30, 40], mask [T, F, T, F] → [10, 0, 30, 0]
let dir = tempdir().unwrap();
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
v.set(0, 10); v.set(1, 20); v.set(2, 30); v.set(3, 40);
v.set(0, 10);
v.set(1, 20);
v.set(2, 30);
v.set(3, 40);
let mut mask = PersistentBitVecBuilder::new(4, &dir.path().join("m.pbiv")).unwrap();
mask.set(0, true); mask.set(2, true);
mask.set(0, true);
mask.set(2, true);
v.mask_with(mask.view());
v.close().unwrap();
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
@@ -132,9 +139,12 @@ fn mask_with_overflow_slot_zeroed() {
// overflow slot (value 500) masked out → removed from overflow, primary=0
let dir = tempdir().unwrap();
let mut v = PersistentCompactIntVecBuilder::new(3, &dir.path().join("v.pciv")).unwrap();
v.set(0, 10); v.set(1, 500); v.set(2, 5);
v.set(0, 10);
v.set(1, 500);
v.set(2, 5);
let mut mask = PersistentBitVecBuilder::new(3, &dir.path().join("m.pbiv")).unwrap();
mask.set(0, true); mask.set(2, true); // slot 1 masked out
mask.set(0, true);
mask.set(2, true); // slot 1 masked out
v.mask_with(mask.view());
v.close().unwrap();
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
@@ -142,14 +152,20 @@ fn mask_with_overflow_slot_zeroed() {
assert_eq!(r.get(1), 0);
assert_eq!(r.get(2), 5);
let ov: Vec<_> = r.view().overflow_entries().collect();
assert!(ov.is_empty(), "overflow entry for masked-out slot should be gone");
assert!(
ov.is_empty(),
"overflow entry for masked-out slot should be gone"
);
}
#[test]
fn mask_with_all_ones_is_noop() {
let dir = tempdir().unwrap();
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
v.set(0, 300); v.set(1, 1); v.set(2, 0); v.set(3, 42);
v.set(0, 300);
v.set(1, 1);
v.set(2, 0);
v.set(3, 42);
let mask = PersistentBitVecBuilder::new_ones(4, &dir.path().join("m.pbiv")).unwrap();
v.mask_with(mask.view());
v.close().unwrap();
@@ -167,9 +183,9 @@ fn bit_partial_group_presence_count() {
// col0=[T,F,T,F], col1=[T,T,F,F], col2=[F,T,T,F]
// group {0,1,2}: counts = [2, 2, 2, 0]
let (_d, m) = make_bit_matrix(&[
&[true, false, true, false],
&[true, true, false, false],
&[false,true, true, false],
&[true, false, true, false],
&[true, true, false, false],
&[false, true, true, false],
]);
let g = ColGroup::new("g", vec![0, 1, 2]);
let result = m.partial_group_presence_count(&g, 1).unwrap();
@@ -184,10 +200,7 @@ fn bit_partial_group_presence_count() {
#[test]
fn bit_partial_group_any() {
// col0=[T,F,F], col1=[F,F,T], group {0,1}: any = [T, F, T]
let (_d, m) = make_bit_matrix(&[
&[true, false, false],
&[false, false, true],
]);
let (_d, m) = make_bit_matrix(&[&[true, false, false], &[false, false, true]]);
let g = ColGroup::new("g", vec![0, 1]);
let result = m.partial_group_any(&g, 1).unwrap();
assert_eq!(result.get(0), true);
src/obicompactvec/src/tests/intmatrix.rs
+109 -45
View File
@@ -1,9 +1,12 @@
use tempfile::tempdir;
use crate::{pack_compact_int_matrix, pack_sparse_compact_int_matrix, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder, StorageKind};
use crate::traits::CountPartials;
use crate::{
PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder,
PersistentIntMatrix, StorageKind, pack_compact_int_matrix, pack_sparse_compact_int_matrix,
};
fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
let n = cols.first().map_or(0, |c| c.len());
let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
@@ -15,7 +18,7 @@ fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatri
cb.close().unwrap();
}
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
(dir, m)
}
@@ -31,7 +34,7 @@ fn single_col_roundtrip() {
col.close().unwrap();
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 1);
assert_eq!(m.n(), 4);
assert_eq!(&*m.row(0), &[10u32]);
@@ -45,14 +48,18 @@ fn two_cols_roundtrip() {
let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(3, &dir.path().join("counts")).unwrap();
let mut col0 = b.add_col().unwrap();
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3);
col0.set(0, 1);
col0.set(1, 2);
col0.set(2, 3);
col0.close().unwrap();
let mut col1 = b.add_col().unwrap();
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30);
col1.set(0, 10);
col1.set(1, 20);
col1.set(2, 30);
col1.close().unwrap();
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 2);
assert_eq!(&*m.row(0), &[1u32, 10]);
assert_eq!(&*m.row(1), &[2u32, 20]);
@@ -66,7 +73,9 @@ fn resume_continues_n_cols_and_appends_columns() {
let mut b = PersistentCompactIntMatrixBuilder::new(3, &counts).unwrap();
let mut col0 = b.add_col().unwrap();
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3);
col0.set(0, 1);
col0.set(1, 2);
col0.set(2, 3);
col0.close().unwrap();
b.close().unwrap();
@@ -74,11 +83,13 @@ fn resume_continues_n_cols_and_appends_columns() {
assert_eq!(resumed.n(), 3);
assert_eq!(resumed.n_cols(), 1);
let mut col1 = resumed.add_col().unwrap();
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30);
col1.set(0, 10);
col1.set(1, 20);
col1.set(2, 30);
col1.close().unwrap();
resumed.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 2);
assert_eq!(&*m.row(0), &[1u32, 10]);
assert_eq!(&*m.row(1), &[2u32, 20]);
@@ -90,17 +101,21 @@ fn resume_twice_keeps_appending() {
let dir = tempdir().unwrap();
let counts = dir.path().join("counts");
PersistentCompactIntMatrixBuilder::new(2, &counts).unwrap().close().unwrap();
PersistentCompactIntMatrixBuilder::new(2, &counts)
.unwrap()
.close()
.unwrap();
for v in [1u32, 2, 3] {
let mut b = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap();
let mut col = b.add_col().unwrap();
col.set(0, v); col.set(1, v * 10);
col.set(0, v);
col.set(1, v * 10);
col.close().unwrap();
b.close().unwrap();
}
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 3);
assert_eq!(&*m.row(0), &[1u32, 2, 3]);
assert_eq!(&*m.row(1), &[10u32, 20, 30]);
@@ -111,11 +126,12 @@ fn col_accessor() {
let dir = tempdir().unwrap();
let mut b = PersistentCompactIntMatrixBuilder::new(2, &dir.path().join("counts")).unwrap();
let mut col0 = b.add_col().unwrap();
col0.set(0, 5); col0.set(1, 7);
col0.set(0, 5);
col0.set(1, 7);
col0.close().unwrap();
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.col(0).get(0), 5);
assert_eq!(m.col(0).get(1), 7);
}
@@ -126,7 +142,7 @@ fn zero_cols_roundtrip() {
let b = PersistentCompactIntMatrixBuilder::new(10, &dir.path().join("counts")).unwrap();
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 0);
assert_eq!(m.n(), 10);
}
@@ -139,7 +155,9 @@ fn bray_dist_matrix_symmetry_and_diagonal() {
let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]);
let dm = m.bray_dist_matrix();
let n = m.n_cols();
for i in 0..n { assert_eq!(dm[[i, i]], 0.0, "diagonal"); }
for i in 0..n {
assert_eq!(dm[[i, i]], 0.0, "diagonal");
}
for i in 0..n {
for j in 0..n {
assert!((dm[[i, j]] - dm[[j, i]]).abs() < 1e-12, "symmetry");
@@ -174,7 +192,7 @@ fn jaccard_dist_matrix_values_match_pairwise() {
#[test]
fn partial_bray_dist_matrix_consistent() {
let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]);
let sum_min = m.partial_bray_dist_matrix();
let sum_min = m.partial_bray_dist_matrix();
let col_sums = m.sum();
let n = m.n_cols();
@@ -189,7 +207,11 @@ fn partial_bray_dist_matrix_consistent() {
for i in 0..n {
for j in i + 1..n {
let denom = col_sums[i] + col_sums[j];
let dist = if denom == 0 { 0.0 } else { 1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64 };
let dist = if denom == 0 {
0.0
} else {
1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64
};
let expected = m.col(i).bray_dist(m.col(j));
assert!((dist - expected).abs() < 1e-12, "[{i},{j}]");
}
@@ -249,13 +271,18 @@ fn partial_relfreq_bray_matches_full() {
let (_d, m) = make_matrix(&[&[1, 0, 2], &[0, 1, 1], &[1, 1, 0]]);
let col_sums = m.sum();
let partial = m.partial_relfreq_bray_dist_matrix(&col_sums);
let full = m.relfreq_bray_dist_matrix();
let full = m.relfreq_bray_dist_matrix();
let n = m.n_cols();
// partial[i,j] = sum_min_relfreq; full[i,j] = 1 - sum_min_relfreq (off-diagonal only)
for i in 0..n {
for j in 0..n {
if i == j { continue; }
assert!((partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12, "[{i},{j}]");
if i == j {
continue;
}
assert!(
(partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12,
"[{i},{j}]"
);
}
}
}
@@ -265,12 +292,15 @@ fn partial_relfreq_euclidean_matches_full() {
let (_d, m) = make_matrix(&[&[3, 0], &[0, 4], &[1, 1]]);
let col_sums = m.sum();
let partial = m.partial_relfreq_euclidean_dist_matrix(&col_sums);
let full = m.relfreq_euclidean_dist_matrix();
let full = m.relfreq_euclidean_dist_matrix();
let n = m.n_cols();
for i in 0..n {
for j in 0..n {
// partial = squared euclidean; full = sqrt(partial)
assert!((partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12, "[{i},{j}]");
assert!(
(partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12,
"[{i},{j}]"
);
}
}
}
@@ -280,12 +310,15 @@ fn partial_hellinger_matches_full() {
let (_d, m) = make_matrix(&[&[3, 0], &[0, 4], &[1, 1]]);
let col_sums = m.sum();
let partial = m.partial_hellinger_euclidean_dist_matrix(&col_sums);
let full = m.hellinger_dist_matrix();
let full = m.hellinger_dist_matrix();
let n = m.n_cols();
for i in 0..n {
for j in 0..n {
// partial / sqrt(2) gives the Hellinger distance
assert!((partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12, "[{i},{j}]");
assert!(
(partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12,
"[{i},{j}]"
);
}
}
}
@@ -295,7 +328,7 @@ fn col_view_packed_values() {
// Build Columnar with overflow values (≥ 255), pack, reopen as Packed, exercise col_view().
let (dir, _col) = make_matrix(&[&[10, 300, 500], &[200, 50, 1000]]);
pack_compact_int_matrix(&dir.path().join("counts")).unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
// col 0: [10, 300, 500] — two overflow slots
let v0 = m.col_view(0);
@@ -326,10 +359,10 @@ fn col_view_packed_matches_columnar() {
// Re-build in a separate dir so we can pack without touching m_col's files.
let (dir_pack, _) = make_matrix(data);
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap();
let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap();
for c in 0..data.len() {
let col_ref = m_col.col(c);
let col_ref = m_col.col(c);
let col_view = m_pack.col_view(c);
assert_eq!(col_view.len(), col_ref.len());
for s in 0..col_ref.len() {
@@ -337,7 +370,7 @@ fn col_view_packed_matches_columnar() {
}
assert_eq!(col_view.sum(), col_ref.sum(), "col={c} sum");
let mut ov_view: Vec<(usize, u32)> = col_view.overflow_entries().collect();
let mut ov_ref: Vec<(usize, u32)> = col_ref.view().overflow_entries().collect();
let mut ov_ref: Vec<(usize, u32)> = col_ref.view().overflow_entries().collect();
ov_view.sort_unstable_by_key(|&(s, _)| s);
ov_ref.sort_unstable_by_key(|&(s, _)| s);
assert_eq!(ov_view, ov_ref, "col={c} overflow_entries");
@@ -355,7 +388,7 @@ fn nonzero_iter_matches_row() {
let (dir_col, m_col) = make_matrix(data);
let (dir_pack, _) = make_matrix(data);
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap();
let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap();
let slots = [4usize, 0, 3, 1];
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
@@ -390,7 +423,7 @@ fn sparse_roundtrip_matches_columnar() {
let (dir_col, m_col) = make_matrix(data);
let (dir_sparse, _) = make_matrix(data);
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
assert_eq!(m_sparse.n(), m_col.n());
@@ -441,7 +474,7 @@ fn sparse_count_partials_match_dense() {
let (dir_col, m_col) = make_matrix(data);
let (dir_sparse, _) = make_matrix(data);
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
let n = m_col.n_cols();
@@ -461,7 +494,11 @@ fn sparse_count_partials_match_dense() {
let eucl_sparse = m_sparse.partial_euclidean_dist_matrix();
for i in 0..n {
for j in 0..n {
close(eucl_col[[i, j]], eucl_sparse[[i, j]], &format!("partial_euclidean[{i},{j}]"));
close(
eucl_col[[i, j]],
eucl_sparse[[i, j]],
&format!("partial_euclidean[{i},{j}]"),
);
}
}
@@ -469,9 +506,16 @@ fn sparse_count_partials_match_dense() {
// (support shortcut), and >1 (general row-major path).
for threshold in [0u32, 1, 2, 3] {
let (inter_col, union_col) = m_col.partial_threshold_jaccard_dist_matrix(threshold);
let (inter_sparse, union_sparse) = m_sparse.partial_threshold_jaccard_dist_matrix(threshold);
assert_eq!(inter_col, inter_sparse, "partial_threshold_jaccard inter, threshold={threshold}");
assert_eq!(union_col, union_sparse, "partial_threshold_jaccard union, threshold={threshold}");
let (inter_sparse, union_sparse) =
m_sparse.partial_threshold_jaccard_dist_matrix(threshold);
assert_eq!(
inter_col, inter_sparse,
"partial_threshold_jaccard inter, threshold={threshold}"
);
assert_eq!(
union_col, union_sparse,
"partial_threshold_jaccard union, threshold={threshold}"
);
}
// partial_relfreq_bray
@@ -479,7 +523,11 @@ fn sparse_count_partials_match_dense() {
let rfb_sparse = m_sparse.partial_relfreq_bray_dist_matrix(&global);
for i in 0..n {
for j in 0..n {
close(rfb_col[[i, j]], rfb_sparse[[i, j]], &format!("partial_relfreq_bray[{i},{j}]"));
close(
rfb_col[[i, j]],
rfb_sparse[[i, j]],
&format!("partial_relfreq_bray[{i},{j}]"),
);
}
}
@@ -488,7 +536,11 @@ fn sparse_count_partials_match_dense() {
let rfe_sparse = m_sparse.partial_relfreq_euclidean_dist_matrix(&global);
for i in 0..n {
for j in 0..n {
close(rfe_col[[i, j]], rfe_sparse[[i, j]], &format!("partial_relfreq_euclidean[{i},{j}]"));
close(
rfe_col[[i, j]],
rfe_sparse[[i, j]],
&format!("partial_relfreq_euclidean[{i},{j}]"),
);
}
}
@@ -497,7 +549,11 @@ fn sparse_count_partials_match_dense() {
let hel_sparse = m_sparse.partial_hellinger_euclidean_dist_matrix(&global);
for i in 0..n {
for j in 0..n {
close(hel_col[[i, j]], hel_sparse[[i, j]], &format!("partial_hellinger[{i},{j}]"));
close(
hel_col[[i, j]],
hel_sparse[[i, j]],
&format!("partial_hellinger[{i},{j}]"),
);
}
}
@@ -514,7 +570,7 @@ fn sparse_roundtrip_from_packed() {
let (dir_sparse, _) = make_matrix(data);
pack_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
for slot in 0..m_col.n() {
assert_eq!(&*m_sparse.row(slot), &*m_col.row(slot), "slot={slot}");
@@ -540,7 +596,10 @@ fn partial_relfreq_bray_additive_across_split() {
let n = m_full.n_cols();
for i in 0..n {
for j in 0..n {
assert!((combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12, "[{i},{j}]");
assert!(
(combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12,
"[{i},{j}]"
);
}
}
}
@@ -548,10 +607,12 @@ fn partial_relfreq_bray_additive_across_split() {
// ── collect_slots_values tests ────────────────────────────────────────────────────────────
fn make_pciv(counts: &[u32]) -> (tempfile::TempDir, PersistentCompactIntVec) {
let dir = tempdir().unwrap();
let dir = tempdir().unwrap();
let path = dir.path().join("c.pciv");
let mut b = PersistentCompactIntVecBuilder::new(counts.len(), &path).unwrap();
for (i, &v) in counts.iter().enumerate() { b.set(i, v); }
for (i, &v) in counts.iter().enumerate() {
b.set(i, v);
}
b.close().unwrap();
let r = PersistentCompactIntVec::open(&path).unwrap();
(dir, r)
@@ -592,7 +653,10 @@ fn pciv_collect_slots_values_empty() {
fn pciv_collect_slots_values_out_of_bounds_panics() {
let (_dir, v) = make_pciv(&[10u32, 20]);
let result = std::panic::catch_unwind(|| v.collect_slots_values(&[0, 2]));
assert!(result.is_err(), "collect_slots_values should panic on out-of-bounds slot");
assert!(
result.is_err(),
"collect_slots_values should panic on out-of-bounds slot"
);
}
// IntSliceView collect_slots_values (same logic, exercised through the view)
src/obicompactvec/src/tests/sparse_intmatrix.rs
+52 -52
View File
@@ -1,13 +1,14 @@
use tempfile::tempdir;
use crate::{
pack_compact_int_matrix, pack_sparse_compact_int_matrix, ColumnWeights, PersistentCompactIntMatrix,
PersistentCompactIntMatrixBuilder, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
ColumnWeights, PersistentIntMatrix, PersistentCompactIntMatrixBuilder,
PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
pack_compact_int_matrix, pack_sparse_compact_int_matrix,
};
/// Builds a dense `PersistentCompactIntMatrix` from column-major `u32` data
/// — mirrors `tests/intmatrix.rs`'s own `make_matrix` helper.
fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
let n = cols.first().map_or(0, |c| c.len());
let dir = tempdir().unwrap();
let counts_dir = dir.path().join("counts");
@@ -20,17 +21,21 @@ fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix
cb.close().unwrap();
}
b.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
(dir, m)
}
/// Builds a sparse int matrix directly from row-major `u32` data (one
/// slice per row, `n_cols` values each) — mirrors `tests/sparse.rs`'s
/// `make_sparse` helper.
fn make_sparse(rows: &[&[u32]], n_cols: usize) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) {
fn make_sparse(
rows: &[&[u32]],
n_cols: usize,
) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) {
let dir = tempdir().unwrap();
let sparse_dir = dir.path().join("sparse");
let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap();
let mut b =
PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap();
let mut cols = Vec::new();
let mut values = Vec::new();
for row in rows {
@@ -54,11 +59,11 @@ fn basic_roundtrip_singletons_and_multi() {
// shares its *support* (columns {0,1}) with another but has different
// values — the whole point of not deduplicating values.
let rows: Vec<&[u32]> = vec![
&[5, 0, 0, 0], // singleton: genome 0, value 5
&[0, 0, 3, 0], // singleton: genome 2, value 3
&[7, 2, 0, 0], // multi: {0:7, 1:2}
&[0, 0, 4, 9], // multi: {2:4, 3:9}
&[1, 6, 0, 0], // multi: same support {0,1} as row 2, different values
&[5, 0, 0, 0], // singleton: genome 0, value 5
&[0, 0, 3, 0], // singleton: genome 2, value 3
&[7, 2, 0, 0], // multi: {0:7, 1:2}
&[0, 0, 4, 9], // multi: {2:4, 3:9}
&[1, 6, 0, 0], // multi: same support {0,1} as row 2, different values
];
let (_dir, m) = make_sparse(&rows, 4);
assert_eq!(m.n(), 5);
@@ -70,11 +75,7 @@ fn basic_roundtrip_singletons_and_multi() {
#[test]
fn get_matches_row() {
let rows: Vec<&[u32]> = vec![
&[3, 0, 5],
&[0, 8, 0],
&[1, 2, 4],
];
let rows: Vec<&[u32]> = vec![&[3, 0, 5], &[0, 8, 0], &[1, 2, 4]];
let (_dir, m) = make_sparse(&rows, 3);
for (slot, row) in rows.iter().enumerate() {
for (c, &expected) in row.iter().enumerate() {
@@ -85,11 +86,7 @@ fn get_matches_row() {
#[test]
fn fill_row_matches_row() {
let rows: Vec<&[u32]> = vec![
&[9, 0, 2],
&[0, 4, 0],
&[1, 1, 1],
];
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]];
let (_dir, m) = make_sparse(&rows, 3);
let mut buf = vec![0u32; 3];
for slot in 0..3 {
@@ -120,11 +117,7 @@ fn fill_sub_matrix_matches_row() {
#[test]
fn nonzero_iter_matches_row() {
let rows: Vec<&[u32]> = vec![
&[9, 0, 2],
&[0, 4, 0],
&[1, 1, 1],
];
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]];
let (_dir, m) = make_sparse(&rows, 3);
let slots = [2usize, 0, 1];
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
@@ -143,12 +136,7 @@ fn nonzero_iter_matches_row() {
#[test]
fn sum_and_count_nonzero_match_naive() {
let rows: Vec<&[u32]> = vec![
&[9, 0, 2],
&[0, 4, 0],
&[1, 1, 1],
&[0, 0, 6],
];
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1], &[0, 0, 6]];
let (_dir, m) = make_sparse(&rows, 3);
let mut expected_sum = [0u64; 3];
let mut expected_count = [0u64; 3];
@@ -193,14 +181,21 @@ fn reopen_after_close_matches_original() {
let rows: Vec<Vec<u32>> = (0..500)
.map(|i| {
(0..37)
.map(|c| if (i * 7 + c * 3) % 11 == 0 { ((i + c) % 250 + 1) as u32 } else { 0 })
.map(|c| {
if (i * 7 + c * 3) % 11 == 0 {
((i + c) % 250 + 1) as u32
} else {
0
}
})
.collect()
})
.collect();
let dir = tempdir().unwrap();
let sparse_dir = dir.path().join("sparse");
{
let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap();
let mut b =
PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap();
let mut cols = Vec::new();
let mut values = Vec::new();
for row in &rows {
@@ -229,10 +224,7 @@ fn reopen_after_close_matches_original() {
fn overflow_values_roundtrip() {
// Values >= 255 exercise the PersistentCompactIntVec overflow path in
// both the singleton and multi value streams.
let rows: Vec<&[u32]> = vec![
&[1000, 0, 0],
&[0, 50_000, 300],
];
let rows: Vec<&[u32]> = vec![&[1000, 0, 0], &[0, 50_000, 300]];
let (_dir, m) = make_sparse(&rows, 3);
for (slot, &expected) in rows.iter().enumerate() {
assert_eq!(&*m.row(slot), expected, "row {slot}");
@@ -246,14 +238,13 @@ fn overflow_values_roundtrip() {
/// `tests::sparse::pack_sparse_bit_matrix_transposes_columnar_directly`.
#[test]
fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
let cols: &[&[u32]] = &[
&[9, 0, 2, 0],
&[0, 4, 0, 7],
&[1, 1, 300, 1],
];
let cols: &[&[u32]] = &[&[9, 0, 2, 0], &[0, 4, 0, 7], &[1, 1, 300, 1]];
let (dir, dense) = make_dense(cols);
let counts_dir = dir.path().join("counts");
assert!(!counts_dir.join("matrix.pcmx").exists(), "still Columnar before packing");
assert!(
!counts_dir.join("matrix.pcmx").exists(),
"still Columnar before packing"
);
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
let n_cols = dense.n_cols();
@@ -262,9 +253,18 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
assert!(counts_dir.join("singleton_values.pciv").exists());
assert!(!counts_dir.join("meta.json").exists(), "columnar meta.json cleaned up");
assert!(!counts_dir.join("col_000000.pciv").exists(), "columnar column files cleaned up");
assert!(!counts_dir.join("matrix.pcmx").exists(), "never materialised");
assert!(
!counts_dir.join("meta.json").exists(),
"columnar meta.json cleaned up"
);
assert!(
!counts_dir.join("col_000000.pciv").exists(),
"columnar column files cleaned up"
);
assert!(
!counts_dir.join("matrix.pcmx").exists(),
"never materialised"
);
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
assert_eq!(sparse.n(), expected_rows.len());
@@ -282,10 +282,7 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
/// it's still used as the transpose source and cleaned up afterward.
#[test]
fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
let cols: &[&[u32]] = &[
&[9, 0, 2],
&[0, 4, 300],
];
let cols: &[&[u32]] = &[&[9, 0, 2], &[0, 4, 300]];
let (dir, dense) = make_dense(cols);
let counts_dir = dir.path().join("counts");
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
@@ -296,7 +293,10 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
assert!(counts_dir.join("matrix.pcmx").exists());
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
assert!(!counts_dir.join("matrix.pcmx").exists(), "packed intermediate cleaned up");
assert!(
!counts_dir.join("matrix.pcmx").exists(),
"packed intermediate cleaned up"
);
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
assert_eq!(sparse.n_cols(), n_cols);
src/obikindex/src/layer/content_layer.rs
+7 -4
View File
@@ -26,7 +26,7 @@ use std::path::{Path, PathBuf};
use crate::layer::utils::LAYERNAME_SUFFIX;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
use obikseq::CanonicalKmer;
use crate::index::error::OKIResult;
@@ -63,7 +63,7 @@ pub enum KmerLayer {
Count {
dir: PathBuf,
id: usize,
layer: TypedLayer<PersistentCompactIntMatrix>,
layer: TypedLayer<PersistentIntMatrix>,
},
Presence {
dir: PathBuf,
@@ -99,7 +99,7 @@ impl KmerLayer {
already_open => return Ok(already_open),
};
if dir.join(COUNTS_DIR).exists() {
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(&dir)?;
let layer = TypedLayer::<PersistentIntMatrix>::open(&dir)?;
Ok(KmerLayer::Count { dir, id, layer })
} else {
let layer = TypedLayer::<PersistentBitMatrix>::open(&dir)?;
@@ -221,7 +221,10 @@ impl KmerLayer {
/// format-agnostic column-major fetch, delegating to whichever matrix
/// this layer actually holds (see `obicompactvec::PersistentBitMatrix`/
/// `PersistentCompactIntMatrix::nonzero_iter`).
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
match self {
KmerLayer::Presence { layer, .. } => layer.nonzero_iter(slots),
KmerLayer::Count { layer, .. } => layer.nonzero_iter(slots),
src/obikindex/src/layer/tests/typed_layer.rs
+71 -30
View File
@@ -1,6 +1,6 @@
use super::*;
use obicompactvec::PersistentSparseBitMatrixBuilder;
use obikseq::{set_k, Unitig};
use obikseq::{Unitig, set_k};
use obiskio::DEFAULT_BLOCK_BITS;
use tempfile::tempdir;
@@ -13,7 +13,8 @@ fn write_unitigs(dir: &Path, seqs: &[&[u8]]) {
}
fn all_canonical_kmers(dir: &Path) -> Vec<CanonicalKmer> {
UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE)).unwrap()
UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE))
.unwrap()
.iter_indexed_canonical_kmers()
.map(|(kmer, _, _)| kmer)
.collect()
@@ -34,13 +35,17 @@ fn canonical_kmer_iter_matches_reader() {
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAAACGT", b"TTTTGCA"]);
let from_iter: Vec<CanonicalKmer> = obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE))
.unwrap()
.collect();
let from_iter: Vec<CanonicalKmer> =
obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE))
.unwrap()
.collect();
let from_reader: Vec<CanonicalKmer> = all_canonical_kmers(dir.path());
assert_eq!(from_iter.len(), from_reader.len(), "different kmer counts");
assert_eq!(from_iter, from_reader, "CanonicalKmerIter and UnitigFileReader disagree");
assert_eq!(
from_iter, from_reader,
"CanonicalKmerIter and UnitigFileReader disagree"
);
}
// ── Generic `TypedLayer<D>` over dense vs. sparse presence storage ───────────────
@@ -64,9 +69,14 @@ fn presence_layer_generic_over_sparse_matches_dense() {
// kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be
// biologically meaningful, just the same on both sides of the
// dense/sparse comparison below.
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
}).unwrap();
TypedLayer::<PersistentBitMatrix>::build_presence(
dir.path(),
DEFAULT_BLOCK_BITS,
&mode,
n_genomes,
|kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0,
)
.unwrap();
let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
assert!(dense_layer.n_cols() >= 1);
@@ -76,10 +86,13 @@ fn presence_layer_generic_over_sparse_matches_dense() {
// same directory, distinct filenames — see
// `DevDocMD/architecture/siblings.md`'s sparse-matrix section).
let dense_matrix = obicompactvec::PersistentBitMatrix::open(dir.path()).unwrap();
PersistentSparseBitMatrixBuilder::build_from_dense(&dense_matrix, &dir.path().join(PRESENCE_DIR))
.unwrap()
.close()
.unwrap();
PersistentSparseBitMatrixBuilder::build_from_dense(
&dense_matrix,
&dir.path().join(PRESENCE_DIR),
)
.unwrap()
.close()
.unwrap();
let sparse_layer = TypedLayer::<PersistentSparseBitMatrix>::open(dir.path()).unwrap();
@@ -88,7 +101,10 @@ fn presence_layer_generic_over_sparse_matches_dense() {
let n = dense_layer.n();
assert_eq!(n, sparse_layer.n());
let slots: Vec<usize> = (0..n).collect();
assert_eq!(dense_layer.sub_matrix(&slots), sparse_layer.sub_matrix(&slots));
assert_eq!(
dense_layer.sub_matrix(&slots),
sparse_layer.sub_matrix(&slots)
);
// The matrix-agnostic, MPHF-only surface (from the generic
// `impl<D: LayerData> TypedLayer<D>`) must also agree: same kmer set,
@@ -124,15 +140,25 @@ fn count_layer_transparently_reads_sparse_after_pack() {
// Deterministic, arbitrary count function — same shape as the presence
// test's arbitrary predicate, just producing a small count instead of a
// bool.
TypedLayer::<()>::build_with_matrix(dir.path(), DEFAULT_BLOCK_BITS, &mode, false, n_genomes, |kmer| {
(0..n_genomes)
.map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32)
.collect()
})
TypedLayer::<()>::build_with_matrix(
dir.path(),
DEFAULT_BLOCK_BITS,
&mode,
false,
n_genomes,
|kmer| {
(0..n_genomes)
.map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32)
.collect()
},
)
.unwrap();
let dense_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
assert_eq!(dense_layer.storage_kind(), obicompactvec::StorageKind::Columnar);
let dense_layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
assert_eq!(
dense_layer.storage_kind(),
obicompactvec::StorageKind::Columnar
);
let n = dense_layer.n();
let slots: Vec<usize> = (0..n).collect();
let dense_sub = dense_layer.sub_matrix(&slots);
@@ -146,8 +172,11 @@ fn count_layer_transparently_reads_sparse_after_pack() {
// in production.
obicompactvec::pack_sparse_compact_int_matrix(&dir.path().join(COUNTS_DIR)).unwrap();
let sparse_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
assert_eq!(sparse_layer.storage_kind(), obicompactvec::StorageKind::Sparse);
let sparse_layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
assert_eq!(
sparse_layer.storage_kind(),
obicompactvec::StorageKind::Sparse
);
assert_eq!(sparse_layer.n(), n);
assert_eq!(sparse_layer.n_cols(), dense_layer.n_cols());
assert_eq!(sparse_layer.sub_matrix(&slots), dense_sub);
@@ -167,13 +196,20 @@ fn count_layer_reports_count_content_and_columnar_storage() {
set_k(4);
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAAACGT"]);
TypedLayer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
.unwrap();
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
TypedLayer::<PersistentIntMatrix>::build(dir.pat
h(), DEFAUL
T_BLOCK_BITS, &Index
Mode::Exact, |_| 1)
,
).unwrap();
let layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
assert_eq!(layer.content(), LayerContent::Count);
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
assert_eq!(layer.evidence_kind(), crate::layer::mphf_layer::EvidenceKind::Exact);
assert_eq!(
layer.evidence_kind(),
crate::layer::mphf_layer::EvidenceKind::Exact
);
}
#[test]
@@ -181,9 +217,14 @@ fn presence_layer_reports_presence_content_and_columnar_storage() {
set_k(4);
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
}).unwrap();
TypedLayer::<PersistentBitMatrix>::build_presence(
dir.path(),
DEFAULT_BLOCK_BITS,
&IndexMode::Exact,
2,
|kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0,
)
.unwrap();
let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
assert_eq!(layer.content(), LayerContent::Presence);
src/obikindex/src/layer/typed_layer.rs
+17 -11
View File
@@ -1,7 +1,7 @@
// use crate::layer::utils::layer_dir;
use obicompactvec::{
BinaryMatrix, ColBuilder, MatrixBuilder, PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentSparseBitMatrix,
PersistentCompactIntMatrixBuilder, PersistentIntMatrix, PersistentSparseBitMatrix,
};
use obikseq::CanonicalKmer;
use obiskio::{UnitigFileReader, UnitigFileWriter};
@@ -47,10 +47,10 @@ impl LayerData for () {
fn read(&self, _slot: usize) {}
}
impl LayerData for PersistentCompactIntMatrix {
impl LayerData for PersistentIntMatrix {
type Item = Box<[u32]>;
fn open(layer_dir: &Path) -> OKIResult<Self> {
PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)
PersistentIntMatrix::open(layer_dir).map_err(OKIError::Io)
}
fn read(&self, slot: usize) -> Box<[u32]> {
self.row(slot)
@@ -112,7 +112,7 @@ pub trait HasLayerContent {
const CONTENT: LayerContent;
}
impl HasLayerContent for PersistentCompactIntMatrix {
impl HasLayerContent for PersistentIntMatrix {
const CONTENT: LayerContent = LayerContent::Count;
}
@@ -135,9 +135,9 @@ pub trait HasStorageKind {
fn storage_kind(&self) -> obicompactvec::StorageKind;
}
impl HasStorageKind for PersistentCompactIntMatrix {
impl HasStorageKind for PersistentIntMatrix {
fn storage_kind(&self) -> obicompactvec::StorageKind {
PersistentCompactIntMatrix::storage_kind(self)
PersistentIntMatrix::storage_kind(self)
}
}
@@ -342,7 +342,7 @@ impl TypedLayer<()> {
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
impl TypedLayer<PersistentCompactIntMatrix> {
impl TypedLayer<PersistentIntMatrix> {
pub fn build(
out_dir: &Path,
block_bits: u8,
@@ -377,12 +377,12 @@ impl TypedLayer<PersistentCompactIntMatrix> {
// ── Mode 2 — count matrix column append ──────────────────────────────────────
impl TypedLayer<PersistentCompactIntMatrix> {
impl TypedLayer<PersistentIntMatrix> {
pub fn append_genome_column(
layer_dir: &Path,
value_of: impl Fn(usize) -> u32,
) -> OKIResult<()> {
PersistentCompactIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of)
PersistentIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of)
.map_err(OKIError::Io)
}
@@ -418,7 +418,10 @@ impl TypedLayer<PersistentCompactIntMatrix> {
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
/// delegates to `PersistentCompactIntMatrix::nonzero_iter`.
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
self.data.nonzero_iter(slots)
}
}
@@ -477,7 +480,10 @@ impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix + obicompactvec::ColumnWeig
impl TypedLayer<PersistentBitMatrix> {
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
/// delegates to `PersistentBitMatrix::nonzero_iter`.
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
self.data.nonzero_iter(slots)
}
src/obikindexer/src/extensions/private_builder.rs
+9 -10
View File
@@ -24,14 +24,14 @@ use std::fs;
use std::io;
use std::path::Path;
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::prelude::*;
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{KmerLayer, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult};
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
@@ -139,7 +139,9 @@ impl PrivateBuilder for KmerIndex {
mode: &IndexMode,
block_bits: u8,
) -> Result<usize, SKError> {
let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?;
let layer0 = self
.layer0(i)
.map_err(|e| io::Error::other(e.to_string()))?;
let layer0_dir = layer0.dir();
let dedup_path = layer0.dereplicated_superkmers_path();
if !dedup_path.exists() {
@@ -193,15 +195,12 @@ impl PrivateBuilder for KmerIndex {
let n_kmers = if with_counts {
let n = write_graph_as_unitigs(g, layer0_dir)
.map_err(|e| io::Error::other(e.to_string()))?;
TypedLayer::<PersistentCompactIntMatrix>::build(
layer0_dir,
block_bits,
mode,
|kmer| match (&mphf1_opt, &counts1_opt) {
TypedLayer::<PersistentIntMatrix>::build(layer0_dir, block_bits, mode, |kmer| {
match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1,
},
)
}
})
.map_err(|e| io::Error::other(e.to_string()))?;
n
} else {
src/obikmerge/src/partition_merge.rs
+16 -8
View File
@@ -34,7 +34,7 @@ fn presence(mode: MergeMode) -> bool {
mod matrix_builder_tests {
use tempfile::tempdir;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
use super::{ColBuilder, MatrixBuilder};
@@ -90,7 +90,7 @@ mod matrix_builder_tests {
col.close().unwrap();
mb.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
let m = PersistentIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 2);
assert_eq!(&*m.row(0), &[0u32, 7]);
assert_eq!(&*m.row(1), &[0u32, 42]);
@@ -422,8 +422,13 @@ pub(crate) fn merge_partition(
move |data: Pass2Data,
push: &PipelineSender<Result<Pass2Data, PipelineError>>,
delta: &PipelineSender<isize>| {
if let Pass2Data::SrcLayer((col_offset, src_n, unitigs_path, src_layer, _guard)) =
data
if let Pass2Data::SrcLayer((
col_offset,
src_n,
unitigs_path,
src_layer,
_guard,
)) = data
{
// _guard dropped at end of block, releasing the slot.
// `src_layer` (MPHF + matrix) was already opened up
@@ -455,8 +460,10 @@ pub(crate) fn merge_partition(
}
}
if !batch.is_empty() {
push.send(Ok(Pass2Data::RawBatch((col_offset, src_n, src_layer, batch))))
.ok();
push.send(Ok(Pass2Data::RawBatch((
col_offset, src_n, src_layer, batch,
))))
.ok();
count += 1;
}
delta.send(count - 1).ok();
@@ -480,8 +487,9 @@ pub(crate) fn merge_partition(
// per kmer — matches the underlying matrix's own
// column-major layout.
let slots = src_layer.hash_batch(&kmers);
let mut cols: Vec<Vec<u32>> =
(0..src_n).map(|_| Vec::with_capacity(kmers.len())).collect();
let mut cols: Vec<Vec<u32>> = (0..src_n)
.map(|_| Vec::with_capacity(kmers.len()))
.collect();
src_layer.fill_sub_matrix(&slots, &mut cols);
// Membership against dst, grouped by dst layer
src/obikphylo/src/matrix_store.rs
+4 -4
View File
@@ -1,15 +1,15 @@
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
use obikidxcache::LayeredStore;
use obikindex::OKIResult;
use obikindex::layer::open_data;
use obikidxcache::LayeredStore;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikindex::load_meta;
use obikindex::KmerIndex;
use obikindex::load_meta;
impl KmerIndex {
/// Open all count matrices for partition `part`, one per layer.
/// Layers without a `counts/` directory are skipped.
pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentCompactIntMatrix>> {
pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentIntMatrix>> {
let index_dir = self.index_dir(part);
if !index_dir.exists() {
return Ok(LayeredStore::new(vec![]));
src/obikselect/src/select_layer.rs
+23 -12
View File
@@ -11,8 +11,8 @@ use std::io;
use std::path::Path;
use obicompactvec::{
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix,
PersistentCompactIntMatrix, TempBitVec, TempCompactIntVec,
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix, PersistentIntMatrix, TempBitVec,
TempCompactIntVec,
};
use obikindex::layer::{KmerLayer, LayerContent};
use obikindex::{KmerIndex, OKIError, OKIResult};
@@ -39,13 +39,15 @@ impl AggOp {
/// `--aggregate-op`.
pub fn parse(s: &str) -> Result<Self, String> {
match s.to_lowercase().as_str() {
"any" => Ok(AggOp::Any),
"all" => Ok(AggOp::All),
"any" => Ok(AggOp::Any),
"all" => Ok(AggOp::All),
"none" => Ok(AggOp::None),
"sum" => Ok(AggOp::Sum),
"min" => Ok(AggOp::Min),
"max" => Ok(AggOp::Max),
other => Err(format!("unknown aggregation operator: {other}; valid: any, all, none, sum, min, max")),
"sum" => Ok(AggOp::Sum),
"min" => Ok(AggOp::Min),
"max" => Ok(AggOp::Max),
other => Err(format!(
"unknown aggregation operator: {other}; valid: any, all, none, sum, min, max"
)),
}
}
@@ -75,7 +77,11 @@ enum AggResult {
Int(TempCompactIntVec),
}
fn compute_group(mat: &dyn MatrixGroupOps, spec: &OutputCol, threshold: u32) -> io::Result<AggResult> {
fn compute_group(
mat: &dyn MatrixGroupOps,
spec: &OutputCol,
threshold: u32,
) -> io::Result<AggResult> {
let g = ColGroup::new(spec.label.clone(), spec.indices.clone());
Ok(match spec.op {
AggOp::Any => AggResult::Bit(mat.partial_group_any(&g, threshold)?),
@@ -145,18 +151,23 @@ pub(crate) fn select_partition(
let group_mat: Box<dyn MatrixGroupOps> = match src_layer.content() {
LayerContent::Count => {
Box::new(PersistentCompactIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
Box::new(PersistentIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
}
LayerContent::Presence => {
Box::new(PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
}
};
let data_subdir = if output_presence { "presence" } else { "counts" };
let data_subdir = if output_presence {
"presence"
} else {
"counts"
};
let data_dir = dst_layer_dir.join(data_subdir);
fs::create_dir_all(&data_dir).map_err(OKIError::Io)?;
let mut builder = MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?;
let mut builder =
MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?;
for spec in specs {
let r = compute_group(group_mat.as_ref(), spec, threshold).map_err(OKIError::Io)?;
add_result(&mut builder, r).map_err(OKIError::Io)?;