Push zunrplorkwkt #70
@@ -13,20 +13,30 @@ use std::error::Error;
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
use obicompactvec::{PersistentCompactIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder};
|
use obicompactvec::{
|
||||||
|
PersistentIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
|
||||||
|
};
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
fn main() -> Result<(), Box<dyn Error>> {
|
||||||
let layer_dir = std::env::args().nth(1).expect("usage: compare_sparse_dense_count <layer_dir>");
|
let layer_dir = std::env::args()
|
||||||
|
.nth(1)
|
||||||
|
.expect("usage: compare_sparse_dense_count <layer_dir>");
|
||||||
let layer_dir = Path::new(&layer_dir);
|
let layer_dir = Path::new(&layer_dir);
|
||||||
|
|
||||||
let t0 = Instant::now();
|
let t0 = Instant::now();
|
||||||
let dense = PersistentCompactIntMatrix::open(layer_dir)?;
|
let dense = PersistentIntMatrix::open(layer_dir)?;
|
||||||
println!("dense ouverte en {:?} ({} lignes x {} colonnes)", t0.elapsed(), dense.n(), dense.n_cols());
|
println!(
|
||||||
|
"dense ouverte en {:?} ({} lignes x {} colonnes)",
|
||||||
|
t0.elapsed(),
|
||||||
|
dense.n(),
|
||||||
|
dense.n_cols()
|
||||||
|
);
|
||||||
|
|
||||||
let out_dir = tempfile::tempdir()?;
|
let out_dir = tempfile::tempdir()?;
|
||||||
|
|
||||||
let t0 = Instant::now();
|
let t0 = Instant::now();
|
||||||
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?.finish()?;
|
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?
|
||||||
|
.finish()?;
|
||||||
println!("build_from_dense: {:?}", t0.elapsed());
|
println!("build_from_dense: {:?}", t0.elapsed());
|
||||||
|
|
||||||
compare(&dense, &sparse)?;
|
compare(&dense, &sparse)?;
|
||||||
@@ -36,7 +46,10 @@ fn main() -> Result<(), Box<dyn Error>> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactIntMatrix) -> Result<(), Box<dyn Error>> {
|
fn compare(
|
||||||
|
dense: &PersistentIntMatrix,
|
||||||
|
sparse: &PersistentSparseCompactIntMatrix,
|
||||||
|
) -> Result<(), Box<dyn Error>> {
|
||||||
let n = dense.n();
|
let n = dense.n();
|
||||||
let n_cols = dense.n_cols();
|
let n_cols = dense.n_cols();
|
||||||
assert_eq!(n, sparse.n(), "n mismatch");
|
assert_eq!(n, sparse.n(), "n mismatch");
|
||||||
@@ -63,7 +76,10 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
println!("comparaison case-a-case: {:?} ({total_cells} cellules)", t0.elapsed());
|
println!(
|
||||||
|
"comparaison case-a-case: {:?} ({total_cells} cellules)",
|
||||||
|
t0.elapsed()
|
||||||
|
);
|
||||||
|
|
||||||
if let Some((slot, col, d, s)) = first_mismatch {
|
if let Some((slot, col, d, s)) = first_mismatch {
|
||||||
eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}");
|
eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}");
|
||||||
@@ -78,7 +94,7 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI
|
|||||||
/// for what's actually stored) — sparsity, singleton-row fraction, and the
|
/// for what's actually stored) — sparsity, singleton-row fraction, and the
|
||||||
/// value-magnitude buckets this crate's overflow encoding is tuned around
|
/// value-magnitude buckets this crate's overflow encoding is tuned around
|
||||||
/// (< 127, < 255, >= 255).
|
/// (< 127, < 255, >= 255).
|
||||||
fn content_stats(dense: &PersistentCompactIntMatrix) {
|
fn content_stats(dense: &PersistentIntMatrix) {
|
||||||
let n = dense.n();
|
let n = dense.n();
|
||||||
let n_cols = dense.n_cols();
|
let n_cols = dense.n_cols();
|
||||||
let total_cells = n as u64 * n_cols as u64;
|
let total_cells = n as u64 * n_cols as u64;
|
||||||
@@ -128,7 +144,10 @@ fn content_stats(dense: &PersistentCompactIntMatrix) {
|
|||||||
100.0 * singleton_rows as f64 / n as f64
|
100.0 * singleton_rows as f64 / n as f64
|
||||||
);
|
);
|
||||||
if nonzero_cells > 0 {
|
if nonzero_cells > 0 {
|
||||||
println!("valeur moyenne (non-nulles): {:.2}", sum as f64 / nonzero_cells as f64);
|
println!(
|
||||||
|
"valeur moyenne (non-nulles): {:.2}",
|
||||||
|
sum as f64 / nonzero_cells as f64
|
||||||
|
);
|
||||||
println!("valeur max: {max_value}");
|
println!("valeur max: {max_value}");
|
||||||
println!(
|
println!(
|
||||||
"valeurs < 127: {under_127} ({:.3}% des non-nulles)",
|
"valeurs < 127: {under_127} ({:.3}% des non-nulles)",
|
||||||
@@ -162,7 +181,7 @@ fn dir_size(dir: &Path) -> std::io::Result<u64> {
|
|||||||
fn compaction_stats(
|
fn compaction_stats(
|
||||||
dense_layer_dir: &Path,
|
dense_layer_dir: &Path,
|
||||||
sparse_dir: &Path,
|
sparse_dir: &Path,
|
||||||
dense: &PersistentCompactIntMatrix,
|
dense: &PersistentIntMatrix,
|
||||||
_sparse: &PersistentSparseCompactIntMatrix,
|
_sparse: &PersistentSparseCompactIntMatrix,
|
||||||
) -> Result<(), Box<dyn Error>> {
|
) -> Result<(), Box<dyn Error>> {
|
||||||
let dense_size = dir_size(&dense_layer_dir.join("counts"))?;
|
let dense_size = dir_size(&dense_layer_dir.join("counts"))?;
|
||||||
|
|||||||
@@ -5,17 +5,19 @@ use std::path::{Path, PathBuf};
|
|||||||
use memmap2::Mmap;
|
use memmap2::Mmap;
|
||||||
use ndarray::{Array1, Array2};
|
use ndarray::{Array1, Array2};
|
||||||
|
|
||||||
use crate::bitmatrix::{fill_row_generic, par_col_reduce, pairwise2_matrix, pairwise_matrix, row_generic};
|
use crate::bitmatrix::{
|
||||||
|
fill_row_generic, pairwise_matrix, pairwise2_matrix, par_col_reduce, row_generic,
|
||||||
|
};
|
||||||
use crate::builder::PersistentCompactIntVecBuilder;
|
use crate::builder::PersistentCompactIntVecBuilder;
|
||||||
use crate::colgroup::{chunked_presence_count, ColGroup, MatrixGroupOps};
|
use crate::colgroup::{ColGroup, MatrixGroupOps, chunked_presence_count};
|
||||||
use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE};
|
use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE};
|
||||||
use crate::meta::MatrixMeta;
|
use crate::meta::MatrixMeta;
|
||||||
use crate::reader::PersistentCompactIntVec;
|
use crate::reader::PersistentCompactIntVec;
|
||||||
use crate::sparse_intmatrix::{is_present as sparse_is_present, PersistentSparseCompactIntMatrix};
|
use crate::sparse_intmatrix::{PersistentSparseCompactIntMatrix, is_present as sparse_is_present};
|
||||||
use crate::storage_kind::StorageKind;
|
use crate::storage_kind::StorageKind;
|
||||||
use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
|
use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
|
||||||
use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
|
use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
|
||||||
use crate::views::{nonzero_triples, IntSliceView};
|
use crate::views::{IntSliceView, nonzero_triples};
|
||||||
|
|
||||||
pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf {
|
pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf {
|
||||||
dir.join(format!("col_{col:06}.pciv"))
|
dir.join(format!("col_{col:06}.pciv"))
|
||||||
@@ -38,11 +40,17 @@ impl ColumnarCompactIntMatrix {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn n(&self) -> usize { self.n }
|
pub(crate) fn n(&self) -> usize {
|
||||||
|
self.n
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn n_cols(&self) -> usize { self.cols.len() }
|
pub(crate) fn n_cols(&self) -> usize {
|
||||||
|
self.cols.len()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec { &self.cols[c] }
|
pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec {
|
||||||
|
&self.cols[c]
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn row(&self, slot: usize) -> Box<[u32]> {
|
pub(crate) fn row(&self, slot: usize) -> Box<[u32]> {
|
||||||
@@ -63,34 +71,64 @@ impl ColumnarCompactIntMatrix {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
||||||
pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_bray_dist(self.col(j)))
|
pairwise_matrix(self.n_cols(), |i, j| {
|
||||||
|
self.col(i).partial_bray_dist(self.col(j))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
|
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_euclidean_dist(self.col(j)))
|
pairwise_matrix(self.n_cols(), |i, j| {
|
||||||
|
self.col(i).partial_euclidean_dist(self.col(j))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
|
pub(crate) fn partial_threshold_jaccard_dist_matrix(
|
||||||
pairwise2_matrix(self.n_cols(), |i, j| self.col(i).partial_threshold_jaccard_dist(self.col(j), threshold))
|
&self,
|
||||||
|
threshold: u32,
|
||||||
|
) -> (Array2<u64>, Array2<u64>) {
|
||||||
|
pairwise2_matrix(self.n_cols(), |i, j| {
|
||||||
|
self.col(i)
|
||||||
|
.partial_threshold_jaccard_dist(self.col(j), threshold)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols(), |i, j| {
|
pairwise_matrix(self.n_cols(), |i, j| {
|
||||||
self.col(i).partial_relfreq_bray_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col(i).partial_relfreq_bray_dist(
|
||||||
|
self.col(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_relfreq_euclidean_dist_matrix(
|
||||||
|
&self,
|
||||||
|
col_sums: &Array1<u64>,
|
||||||
|
) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols(), |i, j| {
|
pairwise_matrix(self.n_cols(), |i, j| {
|
||||||
self.col(i).partial_relfreq_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col(i).partial_relfreq_euclidean_dist(
|
||||||
|
self.col(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_hellinger_euclidean_dist_matrix(
|
||||||
|
&self,
|
||||||
|
col_sums: &Array1<u64>,
|
||||||
|
) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols(), |i, j| {
|
pairwise_matrix(self.n_cols(), |i, j| {
|
||||||
self.col(i).partial_hellinger_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col(i).partial_hellinger_euclidean_dist(
|
||||||
|
self.col(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> {
|
pub(crate) fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> {
|
||||||
let mut meta = MatrixMeta::load(dir)?;
|
let mut meta = MatrixMeta::load(dir)?;
|
||||||
let mut b = PersistentCompactIntVecBuilder::new(meta.n, &col_path(dir, meta.n_cols))?;
|
let mut b = PersistentCompactIntVecBuilder::new(meta.n, &col_path(dir, meta.n_cols))?;
|
||||||
for slot in 0..meta.n { b.set(slot, value_of(slot)); }
|
for slot in 0..meta.n {
|
||||||
|
b.set(slot, value_of(slot));
|
||||||
|
}
|
||||||
b.close()?;
|
b.close()?;
|
||||||
meta.n_cols += 1;
|
meta.n_cols += 1;
|
||||||
meta.save(dir)
|
meta.save(dir)
|
||||||
@@ -119,7 +157,10 @@ impl PackedCompactIntMatrix {
|
|||||||
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
||||||
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
|
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
|
||||||
if mmap.len() < PCMX_HEADER {
|
if mmap.len() < PCMX_HEADER {
|
||||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "PCMX file too short"));
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"PCMX file too short",
|
||||||
|
));
|
||||||
}
|
}
|
||||||
if &mmap[0..4] != &PCMX_MAGIC {
|
if &mmap[0..4] != &PCMX_MAGIC {
|
||||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PCMX magic"));
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PCMX magic"));
|
||||||
@@ -130,32 +171,51 @@ impl PackedCompactIntMatrix {
|
|||||||
let mut columns = Vec::with_capacity(n_cols);
|
let mut columns = Vec::with_capacity(n_cols);
|
||||||
for c in 0..n_cols {
|
for c in 0..n_cols {
|
||||||
let off_pos = PCMX_HEADER + c * 8;
|
let off_pos = PCMX_HEADER + c * 8;
|
||||||
let col_base = u64::from_le_bytes(mmap[off_pos..off_pos+8].try_into().unwrap()) as usize;
|
let col_base =
|
||||||
let n_ov = u64::from_le_bytes(mmap[col_base+16..col_base+24].try_into().unwrap()) as usize;
|
u64::from_le_bytes(mmap[off_pos..off_pos + 8].try_into().unwrap()) as usize;
|
||||||
let n_pciv = u64::from_le_bytes(mmap[col_base+8..col_base+16].try_into().unwrap()) as usize;
|
let n_ov =
|
||||||
|
u64::from_le_bytes(mmap[col_base + 16..col_base + 24].try_into().unwrap()) as usize;
|
||||||
|
let n_pciv =
|
||||||
|
u64::from_le_bytes(mmap[col_base + 8..col_base + 16].try_into().unwrap()) as usize;
|
||||||
let primary_start = col_base + HEADER_SIZE;
|
let primary_start = col_base + HEADER_SIZE;
|
||||||
let data_offset = primary_start + n_pciv;
|
let data_offset = primary_start + n_pciv;
|
||||||
columns.push(ColInfo { primary_start, data_offset, n_overflow: n_ov });
|
columns.push(ColInfo {
|
||||||
|
primary_start,
|
||||||
|
data_offset,
|
||||||
|
n_overflow: n_ov,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
Ok(Self { mmap, n_rows, n_cols, columns })
|
Ok(Self {
|
||||||
|
mmap,
|
||||||
|
n_rows,
|
||||||
|
n_cols,
|
||||||
|
columns,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> {
|
pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> {
|
||||||
let ci = &self.columns[c];
|
let ci = &self.columns[c];
|
||||||
let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
|
let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
|
||||||
let overflow_raw = &self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE];
|
let overflow_raw =
|
||||||
|
&self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE];
|
||||||
IntSliceView::new(primary, overflow_raw, ci.n_overflow, self.n_rows)
|
IntSliceView::new(primary, overflow_raw, ci.n_overflow, self.n_rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentCompactIntVecBuilder> {
|
pub(crate) fn col_persist(
|
||||||
|
&self,
|
||||||
|
c: usize,
|
||||||
|
path: &Path,
|
||||||
|
) -> io::Result<PersistentCompactIntVecBuilder> {
|
||||||
let view = self.col_view(c);
|
let view = self.col_view(c);
|
||||||
let overflow: std::collections::HashMap<usize, u32> = view.overflow_entries().collect();
|
let overflow: std::collections::HashMap<usize, u32> = view.overflow_entries().collect();
|
||||||
PersistentCompactIntVecBuilder::from_raw_primary(view.primary_bytes(), overflow, path)
|
PersistentCompactIntVecBuilder::from_raw_primary(view.primary_bytes(), overflow, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn get(&self, col: usize, slot: usize) -> u32 { self.col_view(col).get(slot) }
|
pub(crate) fn get(&self, col: usize, slot: usize) -> u32 {
|
||||||
|
self.col_view(col).get(slot)
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
|
pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
|
||||||
@@ -180,27 +240,55 @@ impl PackedCompactIntMatrix {
|
|||||||
// via `PersistentCompactIntVec`. No formula is re-derived here.
|
// via `PersistentCompactIntVec`. No formula is re-derived here.
|
||||||
|
|
||||||
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
||||||
pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_bray_dist(self.col_view(j)))
|
pairwise_matrix(self.n_cols, |i, j| {
|
||||||
|
self.col_view(i).partial_bray_dist(self.col_view(j))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
|
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_euclidean_dist(self.col_view(j)))
|
pairwise_matrix(self.n_cols, |i, j| {
|
||||||
|
self.col_view(i).partial_euclidean_dist(self.col_view(j))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, t: u32) -> (Array2<u64>, Array2<u64>) {
|
pub(crate) fn partial_threshold_jaccard_dist_matrix(
|
||||||
pairwise2_matrix(self.n_cols, |i, j| self.col_view(i).partial_threshold_jaccard_dist(self.col_view(j), t))
|
&self,
|
||||||
|
t: u32,
|
||||||
|
) -> (Array2<u64>, Array2<u64>) {
|
||||||
|
pairwise2_matrix(self.n_cols, |i, j| {
|
||||||
|
self.col_view(i)
|
||||||
|
.partial_threshold_jaccard_dist(self.col_view(j), t)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols, |i, j| {
|
pairwise_matrix(self.n_cols, |i, j| {
|
||||||
self.col_view(i).partial_relfreq_bray_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col_view(i).partial_relfreq_bray_dist(
|
||||||
|
self.col_view(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_relfreq_euclidean_dist_matrix(
|
||||||
|
&self,
|
||||||
|
col_sums: &Array1<u64>,
|
||||||
|
) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols, |i, j| {
|
pairwise_matrix(self.n_cols, |i, j| {
|
||||||
self.col_view(i).partial_relfreq_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col_view(i).partial_relfreq_euclidean_dist(
|
||||||
|
self.col_view(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_hellinger_euclidean_dist_matrix(
|
||||||
|
&self,
|
||||||
|
col_sums: &Array1<u64>,
|
||||||
|
) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols, |i, j| {
|
pairwise_matrix(self.n_cols, |i, j| {
|
||||||
self.col_view(i).partial_hellinger_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col_view(i).partial_hellinger_euclidean_dist(
|
||||||
|
self.col_view(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -238,7 +326,9 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
|||||||
// the columnar files are newer and must be (re-)packed, overwriting the
|
// the columnar files are newer and must be (re-)packed, overwriting the
|
||||||
// stale one — never silently discarded as "leftover cleanup".
|
// stale one — never silently discarded as "leftover cleanup".
|
||||||
if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
|
if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
|
||||||
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
|
for c in 0..meta.n_cols {
|
||||||
|
let _ = fs::remove_file(col_path(dir, c));
|
||||||
|
}
|
||||||
let _ = fs::remove_file(dir.join("meta.json"));
|
let _ = fs::remove_file(dir.join("meta.json"));
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
@@ -250,32 +340,41 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
|||||||
let header_size = (PCMX_HEADER + n_cols * 8) as u64;
|
let header_size = (PCMX_HEADER + n_cols * 8) as u64;
|
||||||
let mut col_offset = header_size;
|
let mut col_offset = header_size;
|
||||||
let mut offsets = Vec::with_capacity(n_cols);
|
let mut offsets = Vec::with_capacity(n_cols);
|
||||||
for &size in &col_sizes { offsets.push(col_offset); col_offset += size; }
|
for &size in &col_sizes {
|
||||||
|
offsets.push(col_offset);
|
||||||
|
col_offset += size;
|
||||||
|
}
|
||||||
let tmp_path = dir.join("matrix.pcmx.tmp");
|
let tmp_path = dir.join("matrix.pcmx.tmp");
|
||||||
let mut out = BufWriter::new(File::create(&tmp_path)?);
|
let mut out = BufWriter::new(File::create(&tmp_path)?);
|
||||||
out.write_all(&PCMX_MAGIC)?;
|
out.write_all(&PCMX_MAGIC)?;
|
||||||
out.write_all(&[0u8; 4])?;
|
out.write_all(&[0u8; 4])?;
|
||||||
out.write_all(&(meta.n as u64).to_le_bytes())?;
|
out.write_all(&(meta.n as u64).to_le_bytes())?;
|
||||||
out.write_all(&(n_cols as u64).to_le_bytes())?;
|
out.write_all(&(n_cols as u64).to_le_bytes())?;
|
||||||
for &off in &offsets { out.write_all(&off.to_le_bytes())?; }
|
for &off in &offsets {
|
||||||
for c in 0..n_cols { io::copy(&mut File::open(col_path(dir, c))?, &mut out)?; }
|
out.write_all(&off.to_le_bytes())?;
|
||||||
|
}
|
||||||
|
for c in 0..n_cols {
|
||||||
|
io::copy(&mut File::open(col_path(dir, c))?, &mut out)?;
|
||||||
|
}
|
||||||
out.flush()?;
|
out.flush()?;
|
||||||
drop(out);
|
drop(out);
|
||||||
fs::rename(&tmp_path, &packed_path)?;
|
fs::rename(&tmp_path, &packed_path)?;
|
||||||
for c in 0..n_cols { fs::remove_file(col_path(dir, c))?; }
|
for c in 0..n_cols {
|
||||||
|
fs::remove_file(col_path(dir, c))?;
|
||||||
|
}
|
||||||
fs::remove_file(dir.join("meta.json"))?;
|
fs::remove_file(dir.join("meta.json"))?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── PersistentCompactIntMatrix — public enum ──────────────────────────────────
|
// ── PersistentCompactIntMatrix — public enum ──────────────────────────────────
|
||||||
|
|
||||||
pub enum PersistentCompactIntMatrix {
|
pub enum PersistentIntMatrix {
|
||||||
Columnar(ColumnarCompactIntMatrix),
|
Columnar(ColumnarCompactIntMatrix),
|
||||||
Packed(PackedCompactIntMatrix),
|
Packed(PackedCompactIntMatrix),
|
||||||
Sparse(PersistentSparseCompactIntMatrix),
|
Sparse(PersistentSparseCompactIntMatrix),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PersistentCompactIntMatrix {
|
impl PersistentIntMatrix {
|
||||||
/// Checks (in order): `counts/matrix.pcmx` → Packed; `counts/meta.json`
|
/// Checks (in order): `counts/matrix.pcmx` → Packed; `counts/meta.json`
|
||||||
/// → Columnar; `counts/singleton_values.pciv` → Sparse. Mirrors
|
/// → Columnar; `counts/singleton_values.pciv` → Sparse. Mirrors
|
||||||
/// `PersistentBitMatrix::open`'s own Packed → Columnar → Sparse
|
/// `PersistentBitMatrix::open`'s own Packed → Columnar → Sparse
|
||||||
@@ -283,17 +382,24 @@ impl PersistentCompactIntMatrix {
|
|||||||
pub fn open(layer_dir: &Path) -> io::Result<Self> {
|
pub fn open(layer_dir: &Path) -> io::Result<Self> {
|
||||||
let counts_dir = layer_dir.join("counts");
|
let counts_dir = layer_dir.join("counts");
|
||||||
if counts_dir.join("matrix.pcmx").exists() {
|
if counts_dir.join("matrix.pcmx").exists() {
|
||||||
return Ok(Self::Packed(PackedCompactIntMatrix::open(&counts_dir.join("matrix.pcmx"))?));
|
return Ok(Self::Packed(PackedCompactIntMatrix::open(
|
||||||
|
&counts_dir.join("matrix.pcmx"),
|
||||||
|
)?));
|
||||||
}
|
}
|
||||||
if MatrixMeta::load(&counts_dir).is_ok() {
|
if MatrixMeta::load(&counts_dir).is_ok() {
|
||||||
return Ok(Self::Columnar(ColumnarCompactIntMatrix::open(&counts_dir)?));
|
return Ok(Self::Columnar(ColumnarCompactIntMatrix::open(&counts_dir)?));
|
||||||
}
|
}
|
||||||
if sparse_is_present(&counts_dir) {
|
if sparse_is_present(&counts_dir) {
|
||||||
return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open(&counts_dir)?));
|
return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open(
|
||||||
|
&counts_dir,
|
||||||
|
)?));
|
||||||
}
|
}
|
||||||
Err(io::Error::new(
|
Err(io::Error::new(
|
||||||
io::ErrorKind::NotFound,
|
io::ErrorKind::NotFound,
|
||||||
format!("no count matrix found in {} — run 'obikmer upgrade'", layer_dir.display()),
|
format!(
|
||||||
|
"no count matrix found in {} — run 'obikmer upgrade'",
|
||||||
|
layer_dir.display()
|
||||||
|
),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -330,11 +436,19 @@ impl PersistentCompactIntMatrix {
|
|||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n(&self) -> usize {
|
pub fn n(&self) -> usize {
|
||||||
match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows, Self::Sparse(m) => m.n() }
|
match self {
|
||||||
|
Self::Columnar(m) => m.n(),
|
||||||
|
Self::Packed(m) => m.n_rows,
|
||||||
|
Self::Sparse(m) => m.n(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n_cols(&self) -> usize {
|
pub fn n_cols(&self) -> usize {
|
||||||
match self { Self::Columnar(m) => m.n_cols(), Self::Packed(m) => m.n_cols, Self::Sparse(m) => m.n_cols() }
|
match self {
|
||||||
|
Self::Columnar(m) => m.n_cols(),
|
||||||
|
Self::Packed(m) => m.n_cols,
|
||||||
|
Self::Sparse(m) => m.n_cols(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
@@ -350,7 +464,9 @@ impl PersistentCompactIntMatrix {
|
|||||||
match self {
|
match self {
|
||||||
Self::Columnar(m) => m.col(c).view(),
|
Self::Columnar(m) => m.col(c).view(),
|
||||||
Self::Packed(m) => m.col_view(c),
|
Self::Packed(m) => m.col_view(c),
|
||||||
Self::Sparse(_) => panic!("col_view() not available on Sparse PersistentCompactIntMatrix"),
|
Self::Sparse(_) => {
|
||||||
|
panic!("col_view() not available on Sparse PersistentCompactIntMatrix")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -358,14 +474,20 @@ impl PersistentCompactIntMatrix {
|
|||||||
match self {
|
match self {
|
||||||
Self::Columnar(m) => PersistentCompactIntVecBuilder::build_from(m.col(c), path),
|
Self::Columnar(m) => PersistentCompactIntVecBuilder::build_from(m.col(c), path),
|
||||||
Self::Packed(m) => m.col_persist(c, path),
|
Self::Packed(m) => m.col_persist(c, path),
|
||||||
Self::Sparse(_) => Err(io::Error::new(io::ErrorKind::Unsupported,
|
Self::Sparse(_) => Err(io::Error::new(
|
||||||
"col_persist not available on Sparse PersistentCompactIntMatrix")),
|
io::ErrorKind::Unsupported,
|
||||||
|
"col_persist not available on Sparse PersistentCompactIntMatrix",
|
||||||
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn row(&self, slot: usize) -> Box<[u32]> {
|
pub fn row(&self, slot: usize) -> Box<[u32]> {
|
||||||
match self { Self::Columnar(m) => m.row(slot), Self::Packed(m) => m.row(slot), Self::Sparse(m) => m.row(slot) }
|
match self {
|
||||||
|
Self::Columnar(m) => m.row(slot),
|
||||||
|
Self::Packed(m) => m.row(slot),
|
||||||
|
Self::Sparse(m) => m.row(slot),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
|
pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
|
||||||
@@ -418,18 +540,28 @@ impl PersistentCompactIntMatrix {
|
|||||||
/// `Columnar`/`Packed` reuse the shared sorted-slot batching in
|
/// `Columnar`/`Packed` reuse the shared sorted-slot batching in
|
||||||
/// `views::nonzero_triples`. Boxed, not `impl Iterator`, because the
|
/// `views::nonzero_triples`. Boxed, not `impl Iterator`, because the
|
||||||
/// match arms are genuinely different concrete types.
|
/// match arms are genuinely different concrete types.
|
||||||
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
pub fn nonzero_iter<'a>(
|
||||||
|
&'a self,
|
||||||
|
slots: &'a [usize],
|
||||||
|
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
||||||
match self {
|
match self {
|
||||||
Self::Sparse(m) => Box::new(m.nonzero_iter(slots)),
|
Self::Sparse(m) => Box::new(m.nonzero_iter(slots)),
|
||||||
Self::Columnar(_) | Self::Packed(_) => {
|
Self::Columnar(_) | Self::Packed(_) => Box::new(nonzero_triples(
|
||||||
Box::new(nonzero_triples(slots, self.n_cols(), |c| self.col_view(c), false))
|
slots,
|
||||||
}
|
self.n_cols(),
|
||||||
|
|c| self.col_view(c),
|
||||||
|
false,
|
||||||
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn sum(&self) -> Array1<u64> {
|
pub fn sum(&self) -> Array1<u64> {
|
||||||
match self { Self::Columnar(m) => m.sum(), Self::Packed(m) => m.sum(), Self::Sparse(m) => m.sum() }
|
match self {
|
||||||
|
Self::Columnar(m) => m.sum(),
|
||||||
|
Self::Packed(m) => m.sum(),
|
||||||
|
Self::Sparse(m) => m.sum(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn count_nonzero(&self) -> Array1<u64> {
|
pub fn count_nonzero(&self) -> Array1<u64> {
|
||||||
@@ -456,7 +588,10 @@ impl PersistentCompactIntMatrix {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
|
pub fn partial_threshold_jaccard_dist_matrix(
|
||||||
|
&self,
|
||||||
|
threshold: u32,
|
||||||
|
) -> (Array2<u64>, Array2<u64>) {
|
||||||
match self {
|
match self {
|
||||||
Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
|
Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
|
||||||
Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
|
Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
|
||||||
@@ -497,26 +632,42 @@ impl PersistentCompactIntMatrix {
|
|||||||
|
|
||||||
use crate::traits::{ColumnWeights, CountPartials};
|
use crate::traits::{ColumnWeights, CountPartials};
|
||||||
|
|
||||||
impl ColumnWeights for PersistentCompactIntMatrix {
|
impl ColumnWeights for PersistentIntMatrix {
|
||||||
#[inline]
|
#[inline]
|
||||||
fn col_weights(&self) -> Array1<u64> { self.sum() }
|
fn col_weights(&self) -> Array1<u64> {
|
||||||
|
self.sum()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() }
|
fn partial_kmer_counts(&self) -> Array1<u64> {
|
||||||
|
self.count_nonzero()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CountPartials for PersistentCompactIntMatrix {
|
impl CountPartials for PersistentIntMatrix {
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_bray(&self) -> Array2<u64> { self.partial_bray_dist_matrix() }
|
fn partial_bray(&self) -> Array2<u64> {
|
||||||
|
self.partial_bray_dist_matrix()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_euclidean(&self) -> Array2<f64> { self.partial_euclidean_dist_matrix() }
|
fn partial_euclidean(&self) -> Array2<f64> {
|
||||||
|
self.partial_euclidean_dist_matrix()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) { self.partial_threshold_jaccard_dist_matrix(t) }
|
fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) {
|
||||||
|
self.partial_threshold_jaccard_dist_matrix(t)
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_bray_dist_matrix(g) }
|
fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> {
|
||||||
|
self.partial_relfreq_bray_dist_matrix(g)
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_euclidean_dist_matrix(g) }
|
fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> {
|
||||||
|
self.partial_relfreq_euclidean_dist_matrix(g)
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_hellinger_euclidean_dist_matrix(g) }
|
fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> {
|
||||||
|
self.partial_hellinger_euclidean_dist_matrix(g)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Builder ───────────────────────────────────────────────────────────────────
|
// ── Builder ───────────────────────────────────────────────────────────────────
|
||||||
@@ -530,7 +681,11 @@ pub struct PersistentCompactIntMatrixBuilder {
|
|||||||
impl PersistentCompactIntMatrixBuilder {
|
impl PersistentCompactIntMatrixBuilder {
|
||||||
pub fn new(n: usize, dir: &Path) -> io::Result<Self> {
|
pub fn new(n: usize, dir: &Path) -> io::Result<Self> {
|
||||||
fs::create_dir_all(dir)?;
|
fs::create_dir_all(dir)?;
|
||||||
Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
|
Ok(Self {
|
||||||
|
dir: dir.to_path_buf(),
|
||||||
|
n,
|
||||||
|
n_cols: 0,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resume appending columns to a Columnar matrix directory already
|
/// Resume appending columns to a Columnar matrix directory already
|
||||||
@@ -541,13 +696,21 @@ impl PersistentCompactIntMatrixBuilder {
|
|||||||
/// on-disk column naming or `MatrixMeta` themselves.
|
/// on-disk column naming or `MatrixMeta` themselves.
|
||||||
pub fn resume(dir: &Path) -> io::Result<Self> {
|
pub fn resume(dir: &Path) -> io::Result<Self> {
|
||||||
let meta = MatrixMeta::load(dir)?;
|
let meta = MatrixMeta::load(dir)?;
|
||||||
Ok(Self { dir: dir.to_path_buf(), n: meta.n, n_cols: meta.n_cols })
|
Ok(Self {
|
||||||
|
dir: dir.to_path_buf(),
|
||||||
|
n: meta.n,
|
||||||
|
n_cols: meta.n_cols,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n(&self) -> usize { self.n }
|
pub fn n(&self) -> usize {
|
||||||
|
self.n
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n_cols(&self) -> usize { self.n_cols }
|
pub fn n_cols(&self) -> usize {
|
||||||
|
self.n_cols
|
||||||
|
}
|
||||||
pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> {
|
pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> {
|
||||||
let path = col_path(&self.dir, self.n_cols);
|
let path = col_path(&self.dir, self.n_cols);
|
||||||
self.n_cols += 1;
|
self.n_cols += 1;
|
||||||
@@ -569,14 +732,22 @@ impl PersistentCompactIntMatrixBuilder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn close(self) -> io::Result<()> {
|
pub fn close(self) -> io::Result<()> {
|
||||||
MatrixMeta { n: self.n, n_cols: self.n_cols }.save(&self.dir)
|
MatrixMeta {
|
||||||
|
n: self.n,
|
||||||
|
n_cols: self.n_cols,
|
||||||
|
}
|
||||||
|
.save(&self.dir)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── MatrixGroupOps ────────────────────────────────────────────────────────────
|
// ── MatrixGroupOps ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
impl MatrixGroupOps for PersistentCompactIntMatrix {
|
impl MatrixGroupOps for PersistentIntMatrix {
|
||||||
fn partial_group_presence_count(&self, g: &ColGroup, threshold: u32) -> io::Result<TempCompactIntVec> {
|
fn partial_group_presence_count(
|
||||||
|
&self,
|
||||||
|
g: &ColGroup,
|
||||||
|
threshold: u32,
|
||||||
|
) -> io::Result<TempCompactIntVec> {
|
||||||
chunked_presence_count(self.n(), &g.indices, |b, c| {
|
chunked_presence_count(self.n(), &g.indices, |b, c| {
|
||||||
b.inc_predicate_fast(self.col_view(c), |v| v >= threshold)
|
b.inc_predicate_fast(self.col_view(c), |v| v >= threshold)
|
||||||
})
|
})
|
||||||
@@ -585,7 +756,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
|
|||||||
fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
|
fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
|
||||||
let n = self.n();
|
let n = self.n();
|
||||||
let mut result = TempCompactIntVecBuilder::new(n)?;
|
let mut result = TempCompactIntVecBuilder::new(n)?;
|
||||||
for &c in &g.indices { result.add(self.col_view(c)); }
|
for &c in &g.indices {
|
||||||
|
result.add(self.col_view(c));
|
||||||
|
}
|
||||||
result.freeze()
|
result.freeze()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -603,7 +776,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
|
|||||||
let mut result = TempCompactIntVecBuilder::new(n)?;
|
let mut result = TempCompactIntVecBuilder::new(n)?;
|
||||||
if let Some((&first, rest)) = g.indices.split_first() {
|
if let Some((&first, rest)) = g.indices.split_first() {
|
||||||
result.add(self.col_view(first));
|
result.add(self.col_view(first));
|
||||||
for &c in rest { result.min(self.col_view(c)); }
|
for &c in rest {
|
||||||
|
result.min(self.col_view(c));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
result.freeze()
|
result.freeze()
|
||||||
}
|
}
|
||||||
@@ -611,7 +786,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
|
|||||||
fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
|
fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
|
||||||
let n = self.n();
|
let n = self.n();
|
||||||
let mut result = TempCompactIntVecBuilder::new(n)?;
|
let mut result = TempCompactIntVecBuilder::new(n)?;
|
||||||
for &c in &g.indices { result.max(self.col_view(c)); }
|
for &c in &g.indices {
|
||||||
|
result.max(self.col_view(c));
|
||||||
|
}
|
||||||
result.freeze()
|
result.freeze()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,41 +1,50 @@
|
|||||||
mod bitvec;
|
|
||||||
mod bitmatrix;
|
mod bitmatrix;
|
||||||
|
mod bitvec;
|
||||||
mod builder;
|
mod builder;
|
||||||
mod colgroup;
|
mod colgroup;
|
||||||
mod eliasfano;
|
mod eliasfano;
|
||||||
mod fixedintvec;
|
mod fixedintvec;
|
||||||
mod format;
|
mod format;
|
||||||
mod rankselect;
|
|
||||||
mod intmatrix;
|
mod intmatrix;
|
||||||
mod layer_meta;
|
mod layer_meta;
|
||||||
mod matrix_builder;
|
mod matrix_builder;
|
||||||
mod meta;
|
mod meta;
|
||||||
mod mmap_file;
|
mod mmap_file;
|
||||||
|
mod rankselect;
|
||||||
mod reader;
|
mod reader;
|
||||||
mod sparse_intmatrix;
|
mod sparse_intmatrix;
|
||||||
mod storage_kind;
|
mod storage_kind;
|
||||||
mod tempbitvec;
|
mod tempbitvec;
|
||||||
mod tempintvec;
|
mod tempintvec;
|
||||||
mod views;
|
|
||||||
pub mod traits;
|
pub mod traits;
|
||||||
|
mod views;
|
||||||
|
|
||||||
|
pub use bitmatrix::{
|
||||||
|
PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix,
|
||||||
|
PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix,
|
||||||
|
};
|
||||||
pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
|
pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
|
||||||
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
|
|
||||||
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
|
|
||||||
pub use eliasfano::{EliasFano, EliasFanoBuilder};
|
|
||||||
pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix};
|
|
||||||
pub use builder::PersistentCompactIntVecBuilder;
|
pub use builder::PersistentCompactIntVecBuilder;
|
||||||
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
|
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
|
||||||
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
|
pub use eliasfano::{EliasFano, EliasFanoBuilder};
|
||||||
|
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
|
||||||
|
pub use intmatrix::{
|
||||||
|
PersistentCompactIntMatrixBuilder, PersistentIntMatrix, pack_compact_int_matrix,
|
||||||
|
};
|
||||||
|
|
||||||
pub use layer_meta::LayerMeta;
|
pub use layer_meta::LayerMeta;
|
||||||
pub use matrix_builder::{ColBuilder, MatrixBuilder};
|
pub use matrix_builder::{ColBuilder, MatrixBuilder};
|
||||||
pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter};
|
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
|
||||||
pub use sparse_intmatrix::{PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder, pack_sparse_compact_int_matrix};
|
pub use reader::{Iter as CompactIntVecIter, PersistentCompactIntVec};
|
||||||
|
pub use sparse_intmatrix::{
|
||||||
|
PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
|
||||||
|
pack_sparse_compact_int_matrix,
|
||||||
|
};
|
||||||
pub use storage_kind::StorageKind;
|
pub use storage_kind::StorageKind;
|
||||||
pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
|
pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
|
||||||
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
|
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
|
||||||
pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials};
|
pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials};
|
||||||
pub use views::{BitSliceView, BitSliceIter, IntSliceView, IntSliceViewIter};
|
pub use views::{BitSliceIter, BitSliceView, IntSliceView, IntSliceViewIter};
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[path = "tests/mod.rs"]
|
#[path = "tests/mod.rs"]
|
||||||
|
|||||||
@@ -35,11 +35,17 @@ use crate::builder::PersistentCompactIntVecBuilder;
|
|||||||
use crate::eliasfano::{EliasFano, EliasFanoBuilder};
|
use crate::eliasfano::{EliasFano, EliasFanoBuilder};
|
||||||
use crate::reader::PersistentCompactIntVec;
|
use crate::reader::PersistentCompactIntVec;
|
||||||
use crate::traits::{BitPartials, ColumnWeights, CountPartials};
|
use crate::traits::{BitPartials, ColumnWeights, CountPartials};
|
||||||
use crate::{PersistentCompactIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
|
use crate::{PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
|
||||||
|
|
||||||
fn singleton_values_path(dir: &Path) -> PathBuf { dir.join("singleton_values.pciv") }
|
fn singleton_values_path(dir: &Path) -> PathBuf {
|
||||||
fn multi_values_path(dir: &Path) -> PathBuf { dir.join("multi_values.pciv") }
|
dir.join("singleton_values.pciv")
|
||||||
fn multi_offsets_base(dir: &Path) -> PathBuf { dir.join("multi_offsets") }
|
}
|
||||||
|
fn multi_values_path(dir: &Path) -> PathBuf {
|
||||||
|
dir.join("multi_values.pciv")
|
||||||
|
}
|
||||||
|
fn multi_offsets_base(dir: &Path) -> PathBuf {
|
||||||
|
dir.join("multi_offsets")
|
||||||
|
}
|
||||||
|
|
||||||
/// Lightweight disk probe: true iff a sparse count matrix has already been
|
/// Lightweight disk probe: true iff a sparse count matrix has already been
|
||||||
/// written at `dir` — the marker [`PersistentCompactIntMatrix::open`]/
|
/// written at `dir` — the marker [`PersistentCompactIntMatrix::open`]/
|
||||||
@@ -73,9 +79,13 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n(&self) -> usize { self.support.n() }
|
pub fn n(&self) -> usize {
|
||||||
|
self.support.n()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n_cols(&self) -> usize { self.support.n_cols() }
|
pub fn n_cols(&self) -> usize {
|
||||||
|
self.support.n_cols()
|
||||||
|
}
|
||||||
|
|
||||||
/// Calls `f(col, value)` once per non-zero column at `slot`, in
|
/// Calls `f(col, value)` once per non-zero column at `slot`, in
|
||||||
/// ascending column order — the shared decode branch behind every
|
/// ascending column order — the shared decode branch behind every
|
||||||
@@ -104,7 +114,11 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
/// absent).
|
/// absent).
|
||||||
pub fn get(&self, c: usize, slot: usize) -> u32 {
|
pub fn get(&self, c: usize, slot: usize) -> u32 {
|
||||||
let mut found = 0u32;
|
let mut found = 0u32;
|
||||||
self.for_each_cell_in_row(slot, |g, v| if g == c { found = v; });
|
self.for_each_cell_in_row(slot, |g, v| {
|
||||||
|
if g == c {
|
||||||
|
found = v;
|
||||||
|
}
|
||||||
|
});
|
||||||
found
|
found
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,13 +152,17 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
/// Yields every `(idx into slots, col, value)` triple, row by row, in
|
/// Yields every `(idx into slots, col, value)` triple, row by row, in
|
||||||
/// `slots` order — mirrors
|
/// `slots` order — mirrors
|
||||||
/// [`PersistentSparseBitMatrix::nonzero_iter`].
|
/// [`PersistentSparseBitMatrix::nonzero_iter`].
|
||||||
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator<Item = (usize, usize, u32)> + 'a {
|
pub fn nonzero_iter<'a>(
|
||||||
|
&'a self,
|
||||||
|
slots: &'a [usize],
|
||||||
|
) -> impl Iterator<Item = (usize, usize, u32)> + 'a {
|
||||||
let mut next_slot = 0usize;
|
let mut next_slot = 0usize;
|
||||||
let mut cur_idx = 0usize;
|
let mut cur_idx = 0usize;
|
||||||
let mut buf: Vec<(usize, u32)> = Vec::new();
|
let mut buf: Vec<(usize, u32)> = Vec::new();
|
||||||
let mut buf_pos = 0usize;
|
let mut buf_pos = 0usize;
|
||||||
|
|
||||||
std::iter::from_fn(move || loop {
|
std::iter::from_fn(move || {
|
||||||
|
loop {
|
||||||
if buf_pos < buf.len() {
|
if buf_pos < buf.len() {
|
||||||
let (g, v) = buf[buf_pos];
|
let (g, v) = buf[buf_pos];
|
||||||
buf_pos += 1;
|
buf_pos += 1;
|
||||||
@@ -159,6 +177,7 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
buf.clear();
|
buf.clear();
|
||||||
self.for_each_cell_in_row(slot, |g, v| buf.push((g, v)));
|
self.for_each_cell_in_row(slot, |g, v| buf.push((g, v)));
|
||||||
buf_pos = 0;
|
buf_pos = 0;
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -203,7 +222,11 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
||||||
let mut counts = vec![0u64; self.n_cols()];
|
let mut counts = vec![0u64; self.n_cols()];
|
||||||
for slot in 0..self.n() {
|
for slot in 0..self.n() {
|
||||||
self.for_each_cell_in_row(slot, |g, v| if v >= threshold { counts[g] += 1; });
|
self.for_each_cell_in_row(slot, |g, v| {
|
||||||
|
if v >= threshold {
|
||||||
|
counts[g] += 1;
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
Array1::from(counts)
|
Array1::from(counts)
|
||||||
}
|
}
|
||||||
@@ -261,9 +284,13 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
|
|
||||||
impl ColumnWeights for PersistentSparseCompactIntMatrix {
|
impl ColumnWeights for PersistentSparseCompactIntMatrix {
|
||||||
#[inline]
|
#[inline]
|
||||||
fn col_weights(&self) -> Array1<u64> { self.sum() }
|
fn col_weights(&self) -> Array1<u64> {
|
||||||
|
self.sum()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() }
|
fn partial_kmer_counts(&self) -> Array1<u64> {
|
||||||
|
self.count_nonzero()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CountPartials for PersistentSparseCompactIntMatrix {
|
impl CountPartials for PersistentSparseCompactIntMatrix {
|
||||||
@@ -278,7 +305,9 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
|
|||||||
fn partial_bray(&self) -> Array2<u64> {
|
fn partial_bray(&self) -> Array2<u64> {
|
||||||
let mut m = self.row_major_pairwise(|_, a, _, b| (a as u64).min(b as u64));
|
let mut m = self.row_major_pairwise(|_, a, _, b| (a as u64).min(b as u64));
|
||||||
let sum = self.sum();
|
let sum = self.sum();
|
||||||
for i in 0..self.n_cols() { m[[i, i]] = sum[i]; }
|
for i in 0..self.n_cols() {
|
||||||
|
m[[i, i]] = sum[i];
|
||||||
|
}
|
||||||
m
|
m
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -329,7 +358,8 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
|
|||||||
// left to patch.
|
// left to patch.
|
||||||
return BitPartials::partial_jaccard(&self.support);
|
return BitPartials::partial_jaccard(&self.support);
|
||||||
}
|
}
|
||||||
let mut inter = self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64);
|
let mut inter =
|
||||||
|
self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64);
|
||||||
let mut union = Array2::<u64>::zeros((n, n));
|
let mut union = Array2::<u64>::zeros((n, n));
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
@@ -363,7 +393,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
|
|||||||
});
|
});
|
||||||
let sum = self.sum();
|
let sum = self.sum();
|
||||||
for i in 0..self.n_cols() {
|
for i in 0..self.n_cols() {
|
||||||
m[[i, i]] = if global[i] > 0 { sum[i] as f64 / global[i] as f64 } else { 0.0 };
|
m[[i, i]] = if global[i] > 0 {
|
||||||
|
sum[i] as f64 / global[i] as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
}
|
}
|
||||||
m
|
m
|
||||||
}
|
}
|
||||||
@@ -382,9 +416,21 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
|
|||||||
if i != j {
|
if i != j {
|
||||||
let sa = global[i] as f64;
|
let sa = global[i] as f64;
|
||||||
let sb = global[j] as f64;
|
let sb = global[j] as f64;
|
||||||
let sq_a = if sa > 0.0 { sq[i] as f64 / (sa * sa) } else { 0.0 };
|
let sq_a = if sa > 0.0 {
|
||||||
let sq_b = if sb > 0.0 { sq[j] as f64 / (sb * sb) } else { 0.0 };
|
sq[i] as f64 / (sa * sa)
|
||||||
let cross = if sa > 0.0 && sb > 0.0 { dot[[i, j]] / (sa * sb) } else { 0.0 };
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
let sq_b = if sb > 0.0 {
|
||||||
|
sq[j] as f64 / (sb * sb)
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
let cross = if sa > 0.0 && sb > 0.0 {
|
||||||
|
dot[[i, j]] / (sa * sb)
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
m[[i, j]] = sq_a + sq_b - 2.0 * cross;
|
m[[i, j]] = sq_a + sq_b - 2.0 * cross;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -409,7 +455,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
|
|||||||
let sb = global[j] as f64;
|
let sb = global[j] as f64;
|
||||||
let pa_sum = if sa > 0.0 { sum[i] as f64 / sa } else { 0.0 };
|
let pa_sum = if sa > 0.0 { sum[i] as f64 / sa } else { 0.0 };
|
||||||
let pb_sum = if sb > 0.0 { sum[j] as f64 / sb } else { 0.0 };
|
let pb_sum = if sb > 0.0 { sum[j] as f64 / sb } else { 0.0 };
|
||||||
let cross = if sa > 0.0 && sb > 0.0 { sqrt_dot[[i, j]] / (sa * sb).sqrt() } else { 0.0 };
|
let cross = if sa > 0.0 && sb > 0.0 {
|
||||||
|
sqrt_dot[[i, j]] / (sa * sb).sqrt()
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
m[[i, j]] = pa_sum + pb_sum - 2.0 * cross;
|
m[[i, j]] = pa_sum + pb_sum - 2.0 * cross;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -461,7 +511,6 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
|||||||
self.support.push_row(cols);
|
self.support.push_row(cols);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// Builds a sparse matrix from an already-built dense
|
/// Builds a sparse matrix from an already-built dense
|
||||||
/// [`PersistentCompactIntMatrix`] (`Columnar` or `Packed`) — same
|
/// [`PersistentCompactIntMatrix`] (`Columnar` or `Packed`) — same
|
||||||
/// batched-transpose shape as
|
/// batched-transpose shape as
|
||||||
@@ -475,7 +524,7 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
|||||||
/// pairs are appended to its own buffer in strictly ascending column
|
/// pairs are appended to its own buffer in strictly ascending column
|
||||||
/// order "for free", matching `push_row`'s ascending-`cols` contract
|
/// order "for free", matching `push_row`'s ascending-`cols` contract
|
||||||
/// without an explicit sort.
|
/// without an explicit sort.
|
||||||
pub fn build_from_dense(dense: &PersistentCompactIntMatrix, dir: &Path) -> io::Result<Self> {
|
pub fn build_from_dense(dense: &PersistentIntMatrix, dir: &Path) -> io::Result<Self> {
|
||||||
let n = dense.n();
|
let n = dense.n();
|
||||||
let n_cols = dense.n_cols();
|
let n_cols = dense.n_cols();
|
||||||
let mut builder = Self::new(n, n_cols, dir)?;
|
let mut builder = Self::new(n, n_cols, dir)?;
|
||||||
@@ -500,23 +549,34 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn close(self) -> io::Result<()> {
|
pub fn close(self) -> io::Result<()> {
|
||||||
let Self { support, dir, singleton_values, multi_values, multi_row_offsets } = self;
|
let Self {
|
||||||
|
support,
|
||||||
|
dir,
|
||||||
|
singleton_values,
|
||||||
|
multi_values,
|
||||||
|
multi_row_offsets,
|
||||||
|
} = self;
|
||||||
support.close()?;
|
support.close()?;
|
||||||
|
|
||||||
let mut sv = PersistentCompactIntVecBuilder::new(singleton_values.len(), &singleton_values_path(&dir))?;
|
let mut sv = PersistentCompactIntVecBuilder::new(
|
||||||
|
singleton_values.len(),
|
||||||
|
&singleton_values_path(&dir),
|
||||||
|
)?;
|
||||||
for (i, &v) in singleton_values.iter().enumerate() {
|
for (i, &v) in singleton_values.iter().enumerate() {
|
||||||
sv.set(i, v);
|
sv.set(i, v);
|
||||||
}
|
}
|
||||||
sv.close()?;
|
sv.close()?;
|
||||||
|
|
||||||
let mut mv = PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?;
|
let mut mv =
|
||||||
|
PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?;
|
||||||
for (i, &v) in multi_values.iter().enumerate() {
|
for (i, &v) in multi_values.iter().enumerate() {
|
||||||
mv.set(i, v);
|
mv.set(i, v);
|
||||||
}
|
}
|
||||||
mv.close()?;
|
mv.close()?;
|
||||||
|
|
||||||
let universe = multi_values.len() as u64 + 1;
|
let universe = multi_values.len() as u64 + 1;
|
||||||
let mut mo = EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?;
|
let mut mo =
|
||||||
|
EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?;
|
||||||
for &o in &multi_row_offsets {
|
for &o in &multi_row_offsets {
|
||||||
mo.push(o);
|
mo.push(o);
|
||||||
}
|
}
|
||||||
@@ -544,7 +604,7 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
|||||||
/// half-written one). Idempotent — a no-op if `singleton_values.pciv`
|
/// half-written one). Idempotent — a no-op if `singleton_values.pciv`
|
||||||
/// already exists.
|
/// already exists.
|
||||||
pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
||||||
use crate::intmatrix::{col_path, ColumnarCompactIntMatrix, PackedCompactIntMatrix};
|
use crate::intmatrix::{ColumnarCompactIntMatrix, PackedCompactIntMatrix, col_path};
|
||||||
|
|
||||||
if is_present(dir) {
|
if is_present(dir) {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -552,12 +612,12 @@ pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
|||||||
|
|
||||||
let packed_path = dir.join("matrix.pcmx");
|
let packed_path = dir.join("matrix.pcmx");
|
||||||
if packed_path.exists() {
|
if packed_path.exists() {
|
||||||
let dense = PersistentCompactIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?);
|
let dense = PersistentIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?);
|
||||||
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
||||||
drop(dense);
|
drop(dense);
|
||||||
fs::remove_file(&packed_path)?;
|
fs::remove_file(&packed_path)?;
|
||||||
} else {
|
} else {
|
||||||
let dense = PersistentCompactIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?);
|
let dense = PersistentIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?);
|
||||||
let n_cols = dense.n_cols();
|
let n_cols = dense.n_cols();
|
||||||
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
||||||
drop(dense);
|
drop(dense);
|
||||||
|
|||||||
@@ -1,25 +1,26 @@
|
|||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
ColGroup, MatrixGroupOps,
|
ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder,
|
||||||
PersistentBitMatrix, PersistentBitMatrixBuilder,
|
PersistentIntMatrix, PersistentCompactIntMatrixBuilder,
|
||||||
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
|
|
||||||
};
|
};
|
||||||
use crate::{PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder};
|
use crate::{PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder};
|
||||||
|
|
||||||
// ── helpers ───────────────────────────────────────────────────────────────────
|
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
|
fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
|
||||||
let n = cols.first().map_or(0, |c| c.len());
|
let n = cols.first().map_or(0, |c| c.len());
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
|
||||||
for &col in cols {
|
for &col in cols {
|
||||||
let mut cb = b.add_col().unwrap();
|
let mut cb = b.add_col().unwrap();
|
||||||
for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
|
for (slot, &v) in col.iter().enumerate() {
|
||||||
|
cb.set(slot, v);
|
||||||
|
}
|
||||||
cb.close().unwrap();
|
cb.close().unwrap();
|
||||||
}
|
}
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
(dir, m)
|
(dir, m)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -30,7 +31,9 @@ fn make_bit_matrix(cols: &[&[bool]]) -> (tempfile::TempDir, PersistentBitMatrix)
|
|||||||
let mut b = PersistentBitMatrixBuilder::new(n, &presence).unwrap();
|
let mut b = PersistentBitMatrixBuilder::new(n, &presence).unwrap();
|
||||||
for &col in cols {
|
for &col in cols {
|
||||||
let mut cb = b.add_col().unwrap();
|
let mut cb = b.add_col().unwrap();
|
||||||
for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
|
for (slot, &v) in col.iter().enumerate() {
|
||||||
|
cb.set(slot, v);
|
||||||
|
}
|
||||||
cb.close().unwrap();
|
cb.close().unwrap();
|
||||||
}
|
}
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
@@ -115,9 +118,13 @@ fn mask_with_zeros_selected_slots() {
|
|||||||
// count vec [10, 20, 30, 40], mask [T, F, T, F] → [10, 0, 30, 0]
|
// count vec [10, 20, 30, 40], mask [T, F, T, F] → [10, 0, 30, 0]
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
|
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
|
||||||
v.set(0, 10); v.set(1, 20); v.set(2, 30); v.set(3, 40);
|
v.set(0, 10);
|
||||||
|
v.set(1, 20);
|
||||||
|
v.set(2, 30);
|
||||||
|
v.set(3, 40);
|
||||||
let mut mask = PersistentBitVecBuilder::new(4, &dir.path().join("m.pbiv")).unwrap();
|
let mut mask = PersistentBitVecBuilder::new(4, &dir.path().join("m.pbiv")).unwrap();
|
||||||
mask.set(0, true); mask.set(2, true);
|
mask.set(0, true);
|
||||||
|
mask.set(2, true);
|
||||||
v.mask_with(mask.view());
|
v.mask_with(mask.view());
|
||||||
v.close().unwrap();
|
v.close().unwrap();
|
||||||
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
|
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
|
||||||
@@ -132,9 +139,12 @@ fn mask_with_overflow_slot_zeroed() {
|
|||||||
// overflow slot (value 500) masked out → removed from overflow, primary=0
|
// overflow slot (value 500) masked out → removed from overflow, primary=0
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut v = PersistentCompactIntVecBuilder::new(3, &dir.path().join("v.pciv")).unwrap();
|
let mut v = PersistentCompactIntVecBuilder::new(3, &dir.path().join("v.pciv")).unwrap();
|
||||||
v.set(0, 10); v.set(1, 500); v.set(2, 5);
|
v.set(0, 10);
|
||||||
|
v.set(1, 500);
|
||||||
|
v.set(2, 5);
|
||||||
let mut mask = PersistentBitVecBuilder::new(3, &dir.path().join("m.pbiv")).unwrap();
|
let mut mask = PersistentBitVecBuilder::new(3, &dir.path().join("m.pbiv")).unwrap();
|
||||||
mask.set(0, true); mask.set(2, true); // slot 1 masked out
|
mask.set(0, true);
|
||||||
|
mask.set(2, true); // slot 1 masked out
|
||||||
v.mask_with(mask.view());
|
v.mask_with(mask.view());
|
||||||
v.close().unwrap();
|
v.close().unwrap();
|
||||||
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
|
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
|
||||||
@@ -142,14 +152,20 @@ fn mask_with_overflow_slot_zeroed() {
|
|||||||
assert_eq!(r.get(1), 0);
|
assert_eq!(r.get(1), 0);
|
||||||
assert_eq!(r.get(2), 5);
|
assert_eq!(r.get(2), 5);
|
||||||
let ov: Vec<_> = r.view().overflow_entries().collect();
|
let ov: Vec<_> = r.view().overflow_entries().collect();
|
||||||
assert!(ov.is_empty(), "overflow entry for masked-out slot should be gone");
|
assert!(
|
||||||
|
ov.is_empty(),
|
||||||
|
"overflow entry for masked-out slot should be gone"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mask_with_all_ones_is_noop() {
|
fn mask_with_all_ones_is_noop() {
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
|
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
|
||||||
v.set(0, 300); v.set(1, 1); v.set(2, 0); v.set(3, 42);
|
v.set(0, 300);
|
||||||
|
v.set(1, 1);
|
||||||
|
v.set(2, 0);
|
||||||
|
v.set(3, 42);
|
||||||
let mask = PersistentBitVecBuilder::new_ones(4, &dir.path().join("m.pbiv")).unwrap();
|
let mask = PersistentBitVecBuilder::new_ones(4, &dir.path().join("m.pbiv")).unwrap();
|
||||||
v.mask_with(mask.view());
|
v.mask_with(mask.view());
|
||||||
v.close().unwrap();
|
v.close().unwrap();
|
||||||
@@ -184,10 +200,7 @@ fn bit_partial_group_presence_count() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn bit_partial_group_any() {
|
fn bit_partial_group_any() {
|
||||||
// col0=[T,F,F], col1=[F,F,T], group {0,1}: any = [T, F, T]
|
// col0=[T,F,F], col1=[F,F,T], group {0,1}: any = [T, F, T]
|
||||||
let (_d, m) = make_bit_matrix(&[
|
let (_d, m) = make_bit_matrix(&[&[true, false, false], &[false, false, true]]);
|
||||||
&[true, false, false],
|
|
||||||
&[false, false, true],
|
|
||||||
]);
|
|
||||||
let g = ColGroup::new("g", vec![0, 1]);
|
let g = ColGroup::new("g", vec![0, 1]);
|
||||||
let result = m.partial_group_any(&g, 1).unwrap();
|
let result = m.partial_group_any(&g, 1).unwrap();
|
||||||
assert_eq!(result.get(0), true);
|
assert_eq!(result.get(0), true);
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
use crate::{pack_compact_int_matrix, pack_sparse_compact_int_matrix, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder, StorageKind};
|
|
||||||
use crate::traits::CountPartials;
|
use crate::traits::CountPartials;
|
||||||
|
use crate::{
|
||||||
|
PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder,
|
||||||
|
PersistentIntMatrix, StorageKind, pack_compact_int_matrix, pack_sparse_compact_int_matrix,
|
||||||
|
};
|
||||||
|
|
||||||
fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
|
fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
|
||||||
let n = cols.first().map_or(0, |c| c.len());
|
let n = cols.first().map_or(0, |c| c.len());
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
|
||||||
@@ -15,7 +18,7 @@ fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatri
|
|||||||
cb.close().unwrap();
|
cb.close().unwrap();
|
||||||
}
|
}
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
(dir, m)
|
(dir, m)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -31,7 +34,7 @@ fn single_col_roundtrip() {
|
|||||||
col.close().unwrap();
|
col.close().unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 1);
|
assert_eq!(m.n_cols(), 1);
|
||||||
assert_eq!(m.n(), 4);
|
assert_eq!(m.n(), 4);
|
||||||
assert_eq!(&*m.row(0), &[10u32]);
|
assert_eq!(&*m.row(0), &[10u32]);
|
||||||
@@ -45,14 +48,18 @@ fn two_cols_roundtrip() {
|
|||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::new(3, &dir.path().join("counts")).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::new(3, &dir.path().join("counts")).unwrap();
|
||||||
let mut col0 = b.add_col().unwrap();
|
let mut col0 = b.add_col().unwrap();
|
||||||
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3);
|
col0.set(0, 1);
|
||||||
|
col0.set(1, 2);
|
||||||
|
col0.set(2, 3);
|
||||||
col0.close().unwrap();
|
col0.close().unwrap();
|
||||||
let mut col1 = b.add_col().unwrap();
|
let mut col1 = b.add_col().unwrap();
|
||||||
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30);
|
col1.set(0, 10);
|
||||||
|
col1.set(1, 20);
|
||||||
|
col1.set(2, 30);
|
||||||
col1.close().unwrap();
|
col1.close().unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 2);
|
assert_eq!(m.n_cols(), 2);
|
||||||
assert_eq!(&*m.row(0), &[1u32, 10]);
|
assert_eq!(&*m.row(0), &[1u32, 10]);
|
||||||
assert_eq!(&*m.row(1), &[2u32, 20]);
|
assert_eq!(&*m.row(1), &[2u32, 20]);
|
||||||
@@ -66,7 +73,9 @@ fn resume_continues_n_cols_and_appends_columns() {
|
|||||||
|
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::new(3, &counts).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::new(3, &counts).unwrap();
|
||||||
let mut col0 = b.add_col().unwrap();
|
let mut col0 = b.add_col().unwrap();
|
||||||
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3);
|
col0.set(0, 1);
|
||||||
|
col0.set(1, 2);
|
||||||
|
col0.set(2, 3);
|
||||||
col0.close().unwrap();
|
col0.close().unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
|
|
||||||
@@ -74,11 +83,13 @@ fn resume_continues_n_cols_and_appends_columns() {
|
|||||||
assert_eq!(resumed.n(), 3);
|
assert_eq!(resumed.n(), 3);
|
||||||
assert_eq!(resumed.n_cols(), 1);
|
assert_eq!(resumed.n_cols(), 1);
|
||||||
let mut col1 = resumed.add_col().unwrap();
|
let mut col1 = resumed.add_col().unwrap();
|
||||||
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30);
|
col1.set(0, 10);
|
||||||
|
col1.set(1, 20);
|
||||||
|
col1.set(2, 30);
|
||||||
col1.close().unwrap();
|
col1.close().unwrap();
|
||||||
resumed.close().unwrap();
|
resumed.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 2);
|
assert_eq!(m.n_cols(), 2);
|
||||||
assert_eq!(&*m.row(0), &[1u32, 10]);
|
assert_eq!(&*m.row(0), &[1u32, 10]);
|
||||||
assert_eq!(&*m.row(1), &[2u32, 20]);
|
assert_eq!(&*m.row(1), &[2u32, 20]);
|
||||||
@@ -90,17 +101,21 @@ fn resume_twice_keeps_appending() {
|
|||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let counts = dir.path().join("counts");
|
let counts = dir.path().join("counts");
|
||||||
|
|
||||||
PersistentCompactIntMatrixBuilder::new(2, &counts).unwrap().close().unwrap();
|
PersistentCompactIntMatrixBuilder::new(2, &counts)
|
||||||
|
.unwrap()
|
||||||
|
.close()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
for v in [1u32, 2, 3] {
|
for v in [1u32, 2, 3] {
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap();
|
||||||
let mut col = b.add_col().unwrap();
|
let mut col = b.add_col().unwrap();
|
||||||
col.set(0, v); col.set(1, v * 10);
|
col.set(0, v);
|
||||||
|
col.set(1, v * 10);
|
||||||
col.close().unwrap();
|
col.close().unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 3);
|
assert_eq!(m.n_cols(), 3);
|
||||||
assert_eq!(&*m.row(0), &[1u32, 2, 3]);
|
assert_eq!(&*m.row(0), &[1u32, 2, 3]);
|
||||||
assert_eq!(&*m.row(1), &[10u32, 20, 30]);
|
assert_eq!(&*m.row(1), &[10u32, 20, 30]);
|
||||||
@@ -111,11 +126,12 @@ fn col_accessor() {
|
|||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::new(2, &dir.path().join("counts")).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::new(2, &dir.path().join("counts")).unwrap();
|
||||||
let mut col0 = b.add_col().unwrap();
|
let mut col0 = b.add_col().unwrap();
|
||||||
col0.set(0, 5); col0.set(1, 7);
|
col0.set(0, 5);
|
||||||
|
col0.set(1, 7);
|
||||||
col0.close().unwrap();
|
col0.close().unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.col(0).get(0), 5);
|
assert_eq!(m.col(0).get(0), 5);
|
||||||
assert_eq!(m.col(0).get(1), 7);
|
assert_eq!(m.col(0).get(1), 7);
|
||||||
}
|
}
|
||||||
@@ -126,7 +142,7 @@ fn zero_cols_roundtrip() {
|
|||||||
let b = PersistentCompactIntMatrixBuilder::new(10, &dir.path().join("counts")).unwrap();
|
let b = PersistentCompactIntMatrixBuilder::new(10, &dir.path().join("counts")).unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 0);
|
assert_eq!(m.n_cols(), 0);
|
||||||
assert_eq!(m.n(), 10);
|
assert_eq!(m.n(), 10);
|
||||||
}
|
}
|
||||||
@@ -139,7 +155,9 @@ fn bray_dist_matrix_symmetry_and_diagonal() {
|
|||||||
let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]);
|
let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]);
|
||||||
let dm = m.bray_dist_matrix();
|
let dm = m.bray_dist_matrix();
|
||||||
let n = m.n_cols();
|
let n = m.n_cols();
|
||||||
for i in 0..n { assert_eq!(dm[[i, i]], 0.0, "diagonal"); }
|
for i in 0..n {
|
||||||
|
assert_eq!(dm[[i, i]], 0.0, "diagonal");
|
||||||
|
}
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
assert!((dm[[i, j]] - dm[[j, i]]).abs() < 1e-12, "symmetry");
|
assert!((dm[[i, j]] - dm[[j, i]]).abs() < 1e-12, "symmetry");
|
||||||
@@ -189,7 +207,11 @@ fn partial_bray_dist_matrix_consistent() {
|
|||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in i + 1..n {
|
for j in i + 1..n {
|
||||||
let denom = col_sums[i] + col_sums[j];
|
let denom = col_sums[i] + col_sums[j];
|
||||||
let dist = if denom == 0 { 0.0 } else { 1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64 };
|
let dist = if denom == 0 {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64
|
||||||
|
};
|
||||||
let expected = m.col(i).bray_dist(m.col(j));
|
let expected = m.col(i).bray_dist(m.col(j));
|
||||||
assert!((dist - expected).abs() < 1e-12, "[{i},{j}]");
|
assert!((dist - expected).abs() < 1e-12, "[{i},{j}]");
|
||||||
}
|
}
|
||||||
@@ -254,8 +276,13 @@ fn partial_relfreq_bray_matches_full() {
|
|||||||
// partial[i,j] = sum_min_relfreq; full[i,j] = 1 - sum_min_relfreq (off-diagonal only)
|
// partial[i,j] = sum_min_relfreq; full[i,j] = 1 - sum_min_relfreq (off-diagonal only)
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
if i == j { continue; }
|
if i == j {
|
||||||
assert!((partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12, "[{i},{j}]");
|
continue;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
(partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12,
|
||||||
|
"[{i},{j}]"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -270,7 +297,10 @@ fn partial_relfreq_euclidean_matches_full() {
|
|||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
// partial = squared euclidean; full = sqrt(partial)
|
// partial = squared euclidean; full = sqrt(partial)
|
||||||
assert!((partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12, "[{i},{j}]");
|
assert!(
|
||||||
|
(partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12,
|
||||||
|
"[{i},{j}]"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -285,7 +315,10 @@ fn partial_hellinger_matches_full() {
|
|||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
// partial / sqrt(2) gives the Hellinger distance
|
// partial / sqrt(2) gives the Hellinger distance
|
||||||
assert!((partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12, "[{i},{j}]");
|
assert!(
|
||||||
|
(partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12,
|
||||||
|
"[{i},{j}]"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -295,7 +328,7 @@ fn col_view_packed_values() {
|
|||||||
// Build Columnar with overflow values (≥ 255), pack, reopen as Packed, exercise col_view().
|
// Build Columnar with overflow values (≥ 255), pack, reopen as Packed, exercise col_view().
|
||||||
let (dir, _col) = make_matrix(&[&[10, 300, 500], &[200, 50, 1000]]);
|
let (dir, _col) = make_matrix(&[&[10, 300, 500], &[200, 50, 1000]]);
|
||||||
pack_compact_int_matrix(&dir.path().join("counts")).unwrap();
|
pack_compact_int_matrix(&dir.path().join("counts")).unwrap();
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
|
|
||||||
// col 0: [10, 300, 500] — two overflow slots
|
// col 0: [10, 300, 500] — two overflow slots
|
||||||
let v0 = m.col_view(0);
|
let v0 = m.col_view(0);
|
||||||
@@ -326,7 +359,7 @@ fn col_view_packed_matches_columnar() {
|
|||||||
// Re-build in a separate dir so we can pack without touching m_col's files.
|
// Re-build in a separate dir so we can pack without touching m_col's files.
|
||||||
let (dir_pack, _) = make_matrix(data);
|
let (dir_pack, _) = make_matrix(data);
|
||||||
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
|
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
|
||||||
let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap();
|
let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap();
|
||||||
|
|
||||||
for c in 0..data.len() {
|
for c in 0..data.len() {
|
||||||
let col_ref = m_col.col(c);
|
let col_ref = m_col.col(c);
|
||||||
@@ -355,7 +388,7 @@ fn nonzero_iter_matches_row() {
|
|||||||
let (dir_col, m_col) = make_matrix(data);
|
let (dir_col, m_col) = make_matrix(data);
|
||||||
let (dir_pack, _) = make_matrix(data);
|
let (dir_pack, _) = make_matrix(data);
|
||||||
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
|
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
|
||||||
let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap();
|
let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap();
|
||||||
|
|
||||||
let slots = [4usize, 0, 3, 1];
|
let slots = [4usize, 0, 3, 1];
|
||||||
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
|
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
|
||||||
@@ -390,7 +423,7 @@ fn sparse_roundtrip_matches_columnar() {
|
|||||||
let (dir_col, m_col) = make_matrix(data);
|
let (dir_col, m_col) = make_matrix(data);
|
||||||
let (dir_sparse, _) = make_matrix(data);
|
let (dir_sparse, _) = make_matrix(data);
|
||||||
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
||||||
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
|
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
|
||||||
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
||||||
|
|
||||||
assert_eq!(m_sparse.n(), m_col.n());
|
assert_eq!(m_sparse.n(), m_col.n());
|
||||||
@@ -441,7 +474,7 @@ fn sparse_count_partials_match_dense() {
|
|||||||
let (dir_col, m_col) = make_matrix(data);
|
let (dir_col, m_col) = make_matrix(data);
|
||||||
let (dir_sparse, _) = make_matrix(data);
|
let (dir_sparse, _) = make_matrix(data);
|
||||||
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
||||||
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
|
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
|
||||||
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
||||||
|
|
||||||
let n = m_col.n_cols();
|
let n = m_col.n_cols();
|
||||||
@@ -461,7 +494,11 @@ fn sparse_count_partials_match_dense() {
|
|||||||
let eucl_sparse = m_sparse.partial_euclidean_dist_matrix();
|
let eucl_sparse = m_sparse.partial_euclidean_dist_matrix();
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
close(eucl_col[[i, j]], eucl_sparse[[i, j]], &format!("partial_euclidean[{i},{j}]"));
|
close(
|
||||||
|
eucl_col[[i, j]],
|
||||||
|
eucl_sparse[[i, j]],
|
||||||
|
&format!("partial_euclidean[{i},{j}]"),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -469,9 +506,16 @@ fn sparse_count_partials_match_dense() {
|
|||||||
// (support shortcut), and >1 (general row-major path).
|
// (support shortcut), and >1 (general row-major path).
|
||||||
for threshold in [0u32, 1, 2, 3] {
|
for threshold in [0u32, 1, 2, 3] {
|
||||||
let (inter_col, union_col) = m_col.partial_threshold_jaccard_dist_matrix(threshold);
|
let (inter_col, union_col) = m_col.partial_threshold_jaccard_dist_matrix(threshold);
|
||||||
let (inter_sparse, union_sparse) = m_sparse.partial_threshold_jaccard_dist_matrix(threshold);
|
let (inter_sparse, union_sparse) =
|
||||||
assert_eq!(inter_col, inter_sparse, "partial_threshold_jaccard inter, threshold={threshold}");
|
m_sparse.partial_threshold_jaccard_dist_matrix(threshold);
|
||||||
assert_eq!(union_col, union_sparse, "partial_threshold_jaccard union, threshold={threshold}");
|
assert_eq!(
|
||||||
|
inter_col, inter_sparse,
|
||||||
|
"partial_threshold_jaccard inter, threshold={threshold}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
union_col, union_sparse,
|
||||||
|
"partial_threshold_jaccard union, threshold={threshold}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// partial_relfreq_bray
|
// partial_relfreq_bray
|
||||||
@@ -479,7 +523,11 @@ fn sparse_count_partials_match_dense() {
|
|||||||
let rfb_sparse = m_sparse.partial_relfreq_bray_dist_matrix(&global);
|
let rfb_sparse = m_sparse.partial_relfreq_bray_dist_matrix(&global);
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
close(rfb_col[[i, j]], rfb_sparse[[i, j]], &format!("partial_relfreq_bray[{i},{j}]"));
|
close(
|
||||||
|
rfb_col[[i, j]],
|
||||||
|
rfb_sparse[[i, j]],
|
||||||
|
&format!("partial_relfreq_bray[{i},{j}]"),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -488,7 +536,11 @@ fn sparse_count_partials_match_dense() {
|
|||||||
let rfe_sparse = m_sparse.partial_relfreq_euclidean_dist_matrix(&global);
|
let rfe_sparse = m_sparse.partial_relfreq_euclidean_dist_matrix(&global);
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
close(rfe_col[[i, j]], rfe_sparse[[i, j]], &format!("partial_relfreq_euclidean[{i},{j}]"));
|
close(
|
||||||
|
rfe_col[[i, j]],
|
||||||
|
rfe_sparse[[i, j]],
|
||||||
|
&format!("partial_relfreq_euclidean[{i},{j}]"),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -497,7 +549,11 @@ fn sparse_count_partials_match_dense() {
|
|||||||
let hel_sparse = m_sparse.partial_hellinger_euclidean_dist_matrix(&global);
|
let hel_sparse = m_sparse.partial_hellinger_euclidean_dist_matrix(&global);
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
close(hel_col[[i, j]], hel_sparse[[i, j]], &format!("partial_hellinger[{i},{j}]"));
|
close(
|
||||||
|
hel_col[[i, j]],
|
||||||
|
hel_sparse[[i, j]],
|
||||||
|
&format!("partial_hellinger[{i},{j}]"),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -514,7 +570,7 @@ fn sparse_roundtrip_from_packed() {
|
|||||||
let (dir_sparse, _) = make_matrix(data);
|
let (dir_sparse, _) = make_matrix(data);
|
||||||
pack_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
pack_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
||||||
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
||||||
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
|
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
|
||||||
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
||||||
for slot in 0..m_col.n() {
|
for slot in 0..m_col.n() {
|
||||||
assert_eq!(&*m_sparse.row(slot), &*m_col.row(slot), "slot={slot}");
|
assert_eq!(&*m_sparse.row(slot), &*m_col.row(slot), "slot={slot}");
|
||||||
@@ -540,7 +596,10 @@ fn partial_relfreq_bray_additive_across_split() {
|
|||||||
let n = m_full.n_cols();
|
let n = m_full.n_cols();
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
assert!((combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12, "[{i},{j}]");
|
assert!(
|
||||||
|
(combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12,
|
||||||
|
"[{i},{j}]"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -551,7 +610,9 @@ fn make_pciv(counts: &[u32]) -> (tempfile::TempDir, PersistentCompactIntVec) {
|
|||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let path = dir.path().join("c.pciv");
|
let path = dir.path().join("c.pciv");
|
||||||
let mut b = PersistentCompactIntVecBuilder::new(counts.len(), &path).unwrap();
|
let mut b = PersistentCompactIntVecBuilder::new(counts.len(), &path).unwrap();
|
||||||
for (i, &v) in counts.iter().enumerate() { b.set(i, v); }
|
for (i, &v) in counts.iter().enumerate() {
|
||||||
|
b.set(i, v);
|
||||||
|
}
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
let r = PersistentCompactIntVec::open(&path).unwrap();
|
let r = PersistentCompactIntVec::open(&path).unwrap();
|
||||||
(dir, r)
|
(dir, r)
|
||||||
@@ -592,7 +653,10 @@ fn pciv_collect_slots_values_empty() {
|
|||||||
fn pciv_collect_slots_values_out_of_bounds_panics() {
|
fn pciv_collect_slots_values_out_of_bounds_panics() {
|
||||||
let (_dir, v) = make_pciv(&[10u32, 20]);
|
let (_dir, v) = make_pciv(&[10u32, 20]);
|
||||||
let result = std::panic::catch_unwind(|| v.collect_slots_values(&[0, 2]));
|
let result = std::panic::catch_unwind(|| v.collect_slots_values(&[0, 2]));
|
||||||
assert!(result.is_err(), "collect_slots_values should panic on out-of-bounds slot");
|
assert!(
|
||||||
|
result.is_err(),
|
||||||
|
"collect_slots_values should panic on out-of-bounds slot"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// IntSliceView collect_slots_values (same logic, exercised through the view)
|
// IntSliceView collect_slots_values (same logic, exercised through the view)
|
||||||
|
|||||||
@@ -1,13 +1,14 @@
|
|||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
pack_compact_int_matrix, pack_sparse_compact_int_matrix, ColumnWeights, PersistentCompactIntMatrix,
|
ColumnWeights, PersistentIntMatrix, PersistentCompactIntMatrixBuilder,
|
||||||
PersistentCompactIntMatrixBuilder, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
|
PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
|
||||||
|
pack_compact_int_matrix, pack_sparse_compact_int_matrix,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Builds a dense `PersistentCompactIntMatrix` from column-major `u32` data
|
/// Builds a dense `PersistentCompactIntMatrix` from column-major `u32` data
|
||||||
/// — mirrors `tests/intmatrix.rs`'s own `make_matrix` helper.
|
/// — mirrors `tests/intmatrix.rs`'s own `make_matrix` helper.
|
||||||
fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
|
fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
|
||||||
let n = cols.first().map_or(0, |c| c.len());
|
let n = cols.first().map_or(0, |c| c.len());
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let counts_dir = dir.path().join("counts");
|
let counts_dir = dir.path().join("counts");
|
||||||
@@ -20,17 +21,21 @@ fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix
|
|||||||
cb.close().unwrap();
|
cb.close().unwrap();
|
||||||
}
|
}
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
(dir, m)
|
(dir, m)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Builds a sparse int matrix directly from row-major `u32` data (one
|
/// Builds a sparse int matrix directly from row-major `u32` data (one
|
||||||
/// slice per row, `n_cols` values each) — mirrors `tests/sparse.rs`'s
|
/// slice per row, `n_cols` values each) — mirrors `tests/sparse.rs`'s
|
||||||
/// `make_sparse` helper.
|
/// `make_sparse` helper.
|
||||||
fn make_sparse(rows: &[&[u32]], n_cols: usize) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) {
|
fn make_sparse(
|
||||||
|
rows: &[&[u32]],
|
||||||
|
n_cols: usize,
|
||||||
|
) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) {
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let sparse_dir = dir.path().join("sparse");
|
let sparse_dir = dir.path().join("sparse");
|
||||||
let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap();
|
let mut b =
|
||||||
|
PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap();
|
||||||
let mut cols = Vec::new();
|
let mut cols = Vec::new();
|
||||||
let mut values = Vec::new();
|
let mut values = Vec::new();
|
||||||
for row in rows {
|
for row in rows {
|
||||||
@@ -70,11 +75,7 @@ fn basic_roundtrip_singletons_and_multi() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn get_matches_row() {
|
fn get_matches_row() {
|
||||||
let rows: Vec<&[u32]> = vec![
|
let rows: Vec<&[u32]> = vec![&[3, 0, 5], &[0, 8, 0], &[1, 2, 4]];
|
||||||
&[3, 0, 5],
|
|
||||||
&[0, 8, 0],
|
|
||||||
&[1, 2, 4],
|
|
||||||
];
|
|
||||||
let (_dir, m) = make_sparse(&rows, 3);
|
let (_dir, m) = make_sparse(&rows, 3);
|
||||||
for (slot, row) in rows.iter().enumerate() {
|
for (slot, row) in rows.iter().enumerate() {
|
||||||
for (c, &expected) in row.iter().enumerate() {
|
for (c, &expected) in row.iter().enumerate() {
|
||||||
@@ -85,11 +86,7 @@ fn get_matches_row() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn fill_row_matches_row() {
|
fn fill_row_matches_row() {
|
||||||
let rows: Vec<&[u32]> = vec![
|
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]];
|
||||||
&[9, 0, 2],
|
|
||||||
&[0, 4, 0],
|
|
||||||
&[1, 1, 1],
|
|
||||||
];
|
|
||||||
let (_dir, m) = make_sparse(&rows, 3);
|
let (_dir, m) = make_sparse(&rows, 3);
|
||||||
let mut buf = vec![0u32; 3];
|
let mut buf = vec![0u32; 3];
|
||||||
for slot in 0..3 {
|
for slot in 0..3 {
|
||||||
@@ -120,11 +117,7 @@ fn fill_sub_matrix_matches_row() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn nonzero_iter_matches_row() {
|
fn nonzero_iter_matches_row() {
|
||||||
let rows: Vec<&[u32]> = vec![
|
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]];
|
||||||
&[9, 0, 2],
|
|
||||||
&[0, 4, 0],
|
|
||||||
&[1, 1, 1],
|
|
||||||
];
|
|
||||||
let (_dir, m) = make_sparse(&rows, 3);
|
let (_dir, m) = make_sparse(&rows, 3);
|
||||||
let slots = [2usize, 0, 1];
|
let slots = [2usize, 0, 1];
|
||||||
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
|
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
|
||||||
@@ -143,12 +136,7 @@ fn nonzero_iter_matches_row() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn sum_and_count_nonzero_match_naive() {
|
fn sum_and_count_nonzero_match_naive() {
|
||||||
let rows: Vec<&[u32]> = vec![
|
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1], &[0, 0, 6]];
|
||||||
&[9, 0, 2],
|
|
||||||
&[0, 4, 0],
|
|
||||||
&[1, 1, 1],
|
|
||||||
&[0, 0, 6],
|
|
||||||
];
|
|
||||||
let (_dir, m) = make_sparse(&rows, 3);
|
let (_dir, m) = make_sparse(&rows, 3);
|
||||||
let mut expected_sum = [0u64; 3];
|
let mut expected_sum = [0u64; 3];
|
||||||
let mut expected_count = [0u64; 3];
|
let mut expected_count = [0u64; 3];
|
||||||
@@ -193,14 +181,21 @@ fn reopen_after_close_matches_original() {
|
|||||||
let rows: Vec<Vec<u32>> = (0..500)
|
let rows: Vec<Vec<u32>> = (0..500)
|
||||||
.map(|i| {
|
.map(|i| {
|
||||||
(0..37)
|
(0..37)
|
||||||
.map(|c| if (i * 7 + c * 3) % 11 == 0 { ((i + c) % 250 + 1) as u32 } else { 0 })
|
.map(|c| {
|
||||||
|
if (i * 7 + c * 3) % 11 == 0 {
|
||||||
|
((i + c) % 250 + 1) as u32
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
})
|
||||||
.collect()
|
.collect()
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let sparse_dir = dir.path().join("sparse");
|
let sparse_dir = dir.path().join("sparse");
|
||||||
{
|
{
|
||||||
let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap();
|
let mut b =
|
||||||
|
PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap();
|
||||||
let mut cols = Vec::new();
|
let mut cols = Vec::new();
|
||||||
let mut values = Vec::new();
|
let mut values = Vec::new();
|
||||||
for row in &rows {
|
for row in &rows {
|
||||||
@@ -229,10 +224,7 @@ fn reopen_after_close_matches_original() {
|
|||||||
fn overflow_values_roundtrip() {
|
fn overflow_values_roundtrip() {
|
||||||
// Values >= 255 exercise the PersistentCompactIntVec overflow path in
|
// Values >= 255 exercise the PersistentCompactIntVec overflow path in
|
||||||
// both the singleton and multi value streams.
|
// both the singleton and multi value streams.
|
||||||
let rows: Vec<&[u32]> = vec![
|
let rows: Vec<&[u32]> = vec![&[1000, 0, 0], &[0, 50_000, 300]];
|
||||||
&[1000, 0, 0],
|
|
||||||
&[0, 50_000, 300],
|
|
||||||
];
|
|
||||||
let (_dir, m) = make_sparse(&rows, 3);
|
let (_dir, m) = make_sparse(&rows, 3);
|
||||||
for (slot, &expected) in rows.iter().enumerate() {
|
for (slot, &expected) in rows.iter().enumerate() {
|
||||||
assert_eq!(&*m.row(slot), expected, "row {slot}");
|
assert_eq!(&*m.row(slot), expected, "row {slot}");
|
||||||
@@ -246,14 +238,13 @@ fn overflow_values_roundtrip() {
|
|||||||
/// `tests::sparse::pack_sparse_bit_matrix_transposes_columnar_directly`.
|
/// `tests::sparse::pack_sparse_bit_matrix_transposes_columnar_directly`.
|
||||||
#[test]
|
#[test]
|
||||||
fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
|
fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
|
||||||
let cols: &[&[u32]] = &[
|
let cols: &[&[u32]] = &[&[9, 0, 2, 0], &[0, 4, 0, 7], &[1, 1, 300, 1]];
|
||||||
&[9, 0, 2, 0],
|
|
||||||
&[0, 4, 0, 7],
|
|
||||||
&[1, 1, 300, 1],
|
|
||||||
];
|
|
||||||
let (dir, dense) = make_dense(cols);
|
let (dir, dense) = make_dense(cols);
|
||||||
let counts_dir = dir.path().join("counts");
|
let counts_dir = dir.path().join("counts");
|
||||||
assert!(!counts_dir.join("matrix.pcmx").exists(), "still Columnar before packing");
|
assert!(
|
||||||
|
!counts_dir.join("matrix.pcmx").exists(),
|
||||||
|
"still Columnar before packing"
|
||||||
|
);
|
||||||
|
|
||||||
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
|
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
|
||||||
let n_cols = dense.n_cols();
|
let n_cols = dense.n_cols();
|
||||||
@@ -262,9 +253,18 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
|
|||||||
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
|
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
|
||||||
|
|
||||||
assert!(counts_dir.join("singleton_values.pciv").exists());
|
assert!(counts_dir.join("singleton_values.pciv").exists());
|
||||||
assert!(!counts_dir.join("meta.json").exists(), "columnar meta.json cleaned up");
|
assert!(
|
||||||
assert!(!counts_dir.join("col_000000.pciv").exists(), "columnar column files cleaned up");
|
!counts_dir.join("meta.json").exists(),
|
||||||
assert!(!counts_dir.join("matrix.pcmx").exists(), "never materialised");
|
"columnar meta.json cleaned up"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!counts_dir.join("col_000000.pciv").exists(),
|
||||||
|
"columnar column files cleaned up"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!counts_dir.join("matrix.pcmx").exists(),
|
||||||
|
"never materialised"
|
||||||
|
);
|
||||||
|
|
||||||
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
|
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
|
||||||
assert_eq!(sparse.n(), expected_rows.len());
|
assert_eq!(sparse.n(), expected_rows.len());
|
||||||
@@ -282,10 +282,7 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
|
|||||||
/// it's still used as the transpose source and cleaned up afterward.
|
/// it's still used as the transpose source and cleaned up afterward.
|
||||||
#[test]
|
#[test]
|
||||||
fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
|
fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
|
||||||
let cols: &[&[u32]] = &[
|
let cols: &[&[u32]] = &[&[9, 0, 2], &[0, 4, 300]];
|
||||||
&[9, 0, 2],
|
|
||||||
&[0, 4, 300],
|
|
||||||
];
|
|
||||||
let (dir, dense) = make_dense(cols);
|
let (dir, dense) = make_dense(cols);
|
||||||
let counts_dir = dir.path().join("counts");
|
let counts_dir = dir.path().join("counts");
|
||||||
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
|
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
|
||||||
@@ -296,7 +293,10 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
|
|||||||
assert!(counts_dir.join("matrix.pcmx").exists());
|
assert!(counts_dir.join("matrix.pcmx").exists());
|
||||||
|
|
||||||
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
|
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
|
||||||
assert!(!counts_dir.join("matrix.pcmx").exists(), "packed intermediate cleaned up");
|
assert!(
|
||||||
|
!counts_dir.join("matrix.pcmx").exists(),
|
||||||
|
"packed intermediate cleaned up"
|
||||||
|
);
|
||||||
|
|
||||||
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
|
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
|
||||||
assert_eq!(sparse.n_cols(), n_cols);
|
assert_eq!(sparse.n_cols(), n_cols);
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ use std::path::{Path, PathBuf};
|
|||||||
|
|
||||||
use crate::layer::utils::LAYERNAME_SUFFIX;
|
use crate::layer::utils::LAYERNAME_SUFFIX;
|
||||||
|
|
||||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
|
||||||
use obikseq::CanonicalKmer;
|
use obikseq::CanonicalKmer;
|
||||||
|
|
||||||
use crate::index::error::OKIResult;
|
use crate::index::error::OKIResult;
|
||||||
@@ -63,7 +63,7 @@ pub enum KmerLayer {
|
|||||||
Count {
|
Count {
|
||||||
dir: PathBuf,
|
dir: PathBuf,
|
||||||
id: usize,
|
id: usize,
|
||||||
layer: TypedLayer<PersistentCompactIntMatrix>,
|
layer: TypedLayer<PersistentIntMatrix>,
|
||||||
},
|
},
|
||||||
Presence {
|
Presence {
|
||||||
dir: PathBuf,
|
dir: PathBuf,
|
||||||
@@ -99,7 +99,7 @@ impl KmerLayer {
|
|||||||
already_open => return Ok(already_open),
|
already_open => return Ok(already_open),
|
||||||
};
|
};
|
||||||
if dir.join(COUNTS_DIR).exists() {
|
if dir.join(COUNTS_DIR).exists() {
|
||||||
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(&dir)?;
|
let layer = TypedLayer::<PersistentIntMatrix>::open(&dir)?;
|
||||||
Ok(KmerLayer::Count { dir, id, layer })
|
Ok(KmerLayer::Count { dir, id, layer })
|
||||||
} else {
|
} else {
|
||||||
let layer = TypedLayer::<PersistentBitMatrix>::open(&dir)?;
|
let layer = TypedLayer::<PersistentBitMatrix>::open(&dir)?;
|
||||||
@@ -221,7 +221,10 @@ impl KmerLayer {
|
|||||||
/// format-agnostic column-major fetch, delegating to whichever matrix
|
/// format-agnostic column-major fetch, delegating to whichever matrix
|
||||||
/// this layer actually holds (see `obicompactvec::PersistentBitMatrix`/
|
/// this layer actually holds (see `obicompactvec::PersistentBitMatrix`/
|
||||||
/// `PersistentCompactIntMatrix::nonzero_iter`).
|
/// `PersistentCompactIntMatrix::nonzero_iter`).
|
||||||
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
pub fn nonzero_iter<'a>(
|
||||||
|
&'a self,
|
||||||
|
slots: &'a [usize],
|
||||||
|
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
||||||
match self {
|
match self {
|
||||||
KmerLayer::Presence { layer, .. } => layer.nonzero_iter(slots),
|
KmerLayer::Presence { layer, .. } => layer.nonzero_iter(slots),
|
||||||
KmerLayer::Count { layer, .. } => layer.nonzero_iter(slots),
|
KmerLayer::Count { layer, .. } => layer.nonzero_iter(slots),
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
use obicompactvec::PersistentSparseBitMatrixBuilder;
|
use obicompactvec::PersistentSparseBitMatrixBuilder;
|
||||||
use obikseq::{set_k, Unitig};
|
use obikseq::{Unitig, set_k};
|
||||||
use obiskio::DEFAULT_BLOCK_BITS;
|
use obiskio::DEFAULT_BLOCK_BITS;
|
||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
@@ -13,7 +13,8 @@ fn write_unitigs(dir: &Path, seqs: &[&[u8]]) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn all_canonical_kmers(dir: &Path) -> Vec<CanonicalKmer> {
|
fn all_canonical_kmers(dir: &Path) -> Vec<CanonicalKmer> {
|
||||||
UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE)).unwrap()
|
UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE))
|
||||||
|
.unwrap()
|
||||||
.iter_indexed_canonical_kmers()
|
.iter_indexed_canonical_kmers()
|
||||||
.map(|(kmer, _, _)| kmer)
|
.map(|(kmer, _, _)| kmer)
|
||||||
.collect()
|
.collect()
|
||||||
@@ -34,13 +35,17 @@ fn canonical_kmer_iter_matches_reader() {
|
|||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
write_unitigs(dir.path(), &[b"AAAACGT", b"TTTTGCA"]);
|
write_unitigs(dir.path(), &[b"AAAACGT", b"TTTTGCA"]);
|
||||||
|
|
||||||
let from_iter: Vec<CanonicalKmer> = obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE))
|
let from_iter: Vec<CanonicalKmer> =
|
||||||
|
obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.collect();
|
.collect();
|
||||||
let from_reader: Vec<CanonicalKmer> = all_canonical_kmers(dir.path());
|
let from_reader: Vec<CanonicalKmer> = all_canonical_kmers(dir.path());
|
||||||
|
|
||||||
assert_eq!(from_iter.len(), from_reader.len(), "different kmer counts");
|
assert_eq!(from_iter.len(), from_reader.len(), "different kmer counts");
|
||||||
assert_eq!(from_iter, from_reader, "CanonicalKmerIter and UnitigFileReader disagree");
|
assert_eq!(
|
||||||
|
from_iter, from_reader,
|
||||||
|
"CanonicalKmerIter and UnitigFileReader disagree"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Generic `TypedLayer<D>` over dense vs. sparse presence storage ───────────────
|
// ── Generic `TypedLayer<D>` over dense vs. sparse presence storage ───────────────
|
||||||
@@ -64,9 +69,14 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
|||||||
// kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be
|
// kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be
|
||||||
// biologically meaningful, just the same on both sides of the
|
// biologically meaningful, just the same on both sides of the
|
||||||
// dense/sparse comparison below.
|
// dense/sparse comparison below.
|
||||||
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
|
TypedLayer::<PersistentBitMatrix>::build_presence(
|
||||||
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
dir.path(),
|
||||||
}).unwrap();
|
DEFAULT_BLOCK_BITS,
|
||||||
|
&mode,
|
||||||
|
n_genomes,
|
||||||
|
|kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
|
let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
|
||||||
assert!(dense_layer.n_cols() >= 1);
|
assert!(dense_layer.n_cols() >= 1);
|
||||||
@@ -76,7 +86,10 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
|||||||
// same directory, distinct filenames — see
|
// same directory, distinct filenames — see
|
||||||
// `DevDocMD/architecture/siblings.md`'s sparse-matrix section).
|
// `DevDocMD/architecture/siblings.md`'s sparse-matrix section).
|
||||||
let dense_matrix = obicompactvec::PersistentBitMatrix::open(dir.path()).unwrap();
|
let dense_matrix = obicompactvec::PersistentBitMatrix::open(dir.path()).unwrap();
|
||||||
PersistentSparseBitMatrixBuilder::build_from_dense(&dense_matrix, &dir.path().join(PRESENCE_DIR))
|
PersistentSparseBitMatrixBuilder::build_from_dense(
|
||||||
|
&dense_matrix,
|
||||||
|
&dir.path().join(PRESENCE_DIR),
|
||||||
|
)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.close()
|
.close()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -88,7 +101,10 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
|||||||
let n = dense_layer.n();
|
let n = dense_layer.n();
|
||||||
assert_eq!(n, sparse_layer.n());
|
assert_eq!(n, sparse_layer.n());
|
||||||
let slots: Vec<usize> = (0..n).collect();
|
let slots: Vec<usize> = (0..n).collect();
|
||||||
assert_eq!(dense_layer.sub_matrix(&slots), sparse_layer.sub_matrix(&slots));
|
assert_eq!(
|
||||||
|
dense_layer.sub_matrix(&slots),
|
||||||
|
sparse_layer.sub_matrix(&slots)
|
||||||
|
);
|
||||||
|
|
||||||
// The matrix-agnostic, MPHF-only surface (from the generic
|
// The matrix-agnostic, MPHF-only surface (from the generic
|
||||||
// `impl<D: LayerData> TypedLayer<D>`) must also agree: same kmer set,
|
// `impl<D: LayerData> TypedLayer<D>`) must also agree: same kmer set,
|
||||||
@@ -124,15 +140,25 @@ fn count_layer_transparently_reads_sparse_after_pack() {
|
|||||||
// Deterministic, arbitrary count function — same shape as the presence
|
// Deterministic, arbitrary count function — same shape as the presence
|
||||||
// test's arbitrary predicate, just producing a small count instead of a
|
// test's arbitrary predicate, just producing a small count instead of a
|
||||||
// bool.
|
// bool.
|
||||||
TypedLayer::<()>::build_with_matrix(dir.path(), DEFAULT_BLOCK_BITS, &mode, false, n_genomes, |kmer| {
|
TypedLayer::<()>::build_with_matrix(
|
||||||
|
dir.path(),
|
||||||
|
DEFAULT_BLOCK_BITS,
|
||||||
|
&mode,
|
||||||
|
false,
|
||||||
|
n_genomes,
|
||||||
|
|kmer| {
|
||||||
(0..n_genomes)
|
(0..n_genomes)
|
||||||
.map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32)
|
.map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32)
|
||||||
.collect()
|
.collect()
|
||||||
})
|
},
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let dense_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
|
let dense_layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
|
||||||
assert_eq!(dense_layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
assert_eq!(
|
||||||
|
dense_layer.storage_kind(),
|
||||||
|
obicompactvec::StorageKind::Columnar
|
||||||
|
);
|
||||||
let n = dense_layer.n();
|
let n = dense_layer.n();
|
||||||
let slots: Vec<usize> = (0..n).collect();
|
let slots: Vec<usize> = (0..n).collect();
|
||||||
let dense_sub = dense_layer.sub_matrix(&slots);
|
let dense_sub = dense_layer.sub_matrix(&slots);
|
||||||
@@ -146,8 +172,11 @@ fn count_layer_transparently_reads_sparse_after_pack() {
|
|||||||
// in production.
|
// in production.
|
||||||
obicompactvec::pack_sparse_compact_int_matrix(&dir.path().join(COUNTS_DIR)).unwrap();
|
obicompactvec::pack_sparse_compact_int_matrix(&dir.path().join(COUNTS_DIR)).unwrap();
|
||||||
|
|
||||||
let sparse_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
|
let sparse_layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
|
||||||
assert_eq!(sparse_layer.storage_kind(), obicompactvec::StorageKind::Sparse);
|
assert_eq!(
|
||||||
|
sparse_layer.storage_kind(),
|
||||||
|
obicompactvec::StorageKind::Sparse
|
||||||
|
);
|
||||||
assert_eq!(sparse_layer.n(), n);
|
assert_eq!(sparse_layer.n(), n);
|
||||||
assert_eq!(sparse_layer.n_cols(), dense_layer.n_cols());
|
assert_eq!(sparse_layer.n_cols(), dense_layer.n_cols());
|
||||||
assert_eq!(sparse_layer.sub_matrix(&slots), dense_sub);
|
assert_eq!(sparse_layer.sub_matrix(&slots), dense_sub);
|
||||||
@@ -167,13 +196,20 @@ fn count_layer_reports_count_content_and_columnar_storage() {
|
|||||||
set_k(4);
|
set_k(4);
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
write_unitigs(dir.path(), &[b"AAAACGT"]);
|
write_unitigs(dir.path(), &[b"AAAACGT"]);
|
||||||
TypedLayer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
|
TypedLayer::<PersistentIntMatrix>::build(dir.pat
|
||||||
.unwrap();
|
h(), DEFAUL
|
||||||
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
|
T_BLOCK_BITS, &Index
|
||||||
|
Mode::Exact, |_| 1)
|
||||||
|
,
|
||||||
|
).unwrap();
|
||||||
|
let layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
|
||||||
|
|
||||||
assert_eq!(layer.content(), LayerContent::Count);
|
assert_eq!(layer.content(), LayerContent::Count);
|
||||||
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
||||||
assert_eq!(layer.evidence_kind(), crate::layer::mphf_layer::EvidenceKind::Exact);
|
assert_eq!(
|
||||||
|
layer.evidence_kind(),
|
||||||
|
crate::layer::mphf_layer::EvidenceKind::Exact
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -181,9 +217,14 @@ fn presence_layer_reports_presence_content_and_columnar_storage() {
|
|||||||
set_k(4);
|
set_k(4);
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
|
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
|
||||||
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
|
TypedLayer::<PersistentBitMatrix>::build_presence(
|
||||||
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
dir.path(),
|
||||||
}).unwrap();
|
DEFAULT_BLOCK_BITS,
|
||||||
|
&IndexMode::Exact,
|
||||||
|
2,
|
||||||
|
|kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
|
let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
|
||||||
|
|
||||||
assert_eq!(layer.content(), LayerContent::Presence);
|
assert_eq!(layer.content(), LayerContent::Presence);
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// use crate::layer::utils::layer_dir;
|
// use crate::layer::utils::layer_dir;
|
||||||
use obicompactvec::{
|
use obicompactvec::{
|
||||||
BinaryMatrix, ColBuilder, MatrixBuilder, PersistentBitMatrix, PersistentBitMatrixBuilder,
|
BinaryMatrix, ColBuilder, MatrixBuilder, PersistentBitMatrix, PersistentBitMatrixBuilder,
|
||||||
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentSparseBitMatrix,
|
PersistentCompactIntMatrixBuilder, PersistentIntMatrix, PersistentSparseBitMatrix,
|
||||||
};
|
};
|
||||||
use obikseq::CanonicalKmer;
|
use obikseq::CanonicalKmer;
|
||||||
use obiskio::{UnitigFileReader, UnitigFileWriter};
|
use obiskio::{UnitigFileReader, UnitigFileWriter};
|
||||||
@@ -47,10 +47,10 @@ impl LayerData for () {
|
|||||||
fn read(&self, _slot: usize) {}
|
fn read(&self, _slot: usize) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LayerData for PersistentCompactIntMatrix {
|
impl LayerData for PersistentIntMatrix {
|
||||||
type Item = Box<[u32]>;
|
type Item = Box<[u32]>;
|
||||||
fn open(layer_dir: &Path) -> OKIResult<Self> {
|
fn open(layer_dir: &Path) -> OKIResult<Self> {
|
||||||
PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)
|
PersistentIntMatrix::open(layer_dir).map_err(OKIError::Io)
|
||||||
}
|
}
|
||||||
fn read(&self, slot: usize) -> Box<[u32]> {
|
fn read(&self, slot: usize) -> Box<[u32]> {
|
||||||
self.row(slot)
|
self.row(slot)
|
||||||
@@ -112,7 +112,7 @@ pub trait HasLayerContent {
|
|||||||
const CONTENT: LayerContent;
|
const CONTENT: LayerContent;
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HasLayerContent for PersistentCompactIntMatrix {
|
impl HasLayerContent for PersistentIntMatrix {
|
||||||
const CONTENT: LayerContent = LayerContent::Count;
|
const CONTENT: LayerContent = LayerContent::Count;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -135,9 +135,9 @@ pub trait HasStorageKind {
|
|||||||
fn storage_kind(&self) -> obicompactvec::StorageKind;
|
fn storage_kind(&self) -> obicompactvec::StorageKind;
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HasStorageKind for PersistentCompactIntMatrix {
|
impl HasStorageKind for PersistentIntMatrix {
|
||||||
fn storage_kind(&self) -> obicompactvec::StorageKind {
|
fn storage_kind(&self) -> obicompactvec::StorageKind {
|
||||||
PersistentCompactIntMatrix::storage_kind(self)
|
PersistentIntMatrix::storage_kind(self)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -342,7 +342,7 @@ impl TypedLayer<()> {
|
|||||||
|
|
||||||
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
|
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
impl TypedLayer<PersistentCompactIntMatrix> {
|
impl TypedLayer<PersistentIntMatrix> {
|
||||||
pub fn build(
|
pub fn build(
|
||||||
out_dir: &Path,
|
out_dir: &Path,
|
||||||
block_bits: u8,
|
block_bits: u8,
|
||||||
@@ -377,12 +377,12 @@ impl TypedLayer<PersistentCompactIntMatrix> {
|
|||||||
|
|
||||||
// ── Mode 2 — count matrix column append ──────────────────────────────────────
|
// ── Mode 2 — count matrix column append ──────────────────────────────────────
|
||||||
|
|
||||||
impl TypedLayer<PersistentCompactIntMatrix> {
|
impl TypedLayer<PersistentIntMatrix> {
|
||||||
pub fn append_genome_column(
|
pub fn append_genome_column(
|
||||||
layer_dir: &Path,
|
layer_dir: &Path,
|
||||||
value_of: impl Fn(usize) -> u32,
|
value_of: impl Fn(usize) -> u32,
|
||||||
) -> OKIResult<()> {
|
) -> OKIResult<()> {
|
||||||
PersistentCompactIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of)
|
PersistentIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of)
|
||||||
.map_err(OKIError::Io)
|
.map_err(OKIError::Io)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -418,7 +418,10 @@ impl TypedLayer<PersistentCompactIntMatrix> {
|
|||||||
|
|
||||||
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
|
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
|
||||||
/// delegates to `PersistentCompactIntMatrix::nonzero_iter`.
|
/// delegates to `PersistentCompactIntMatrix::nonzero_iter`.
|
||||||
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
pub fn nonzero_iter<'a>(
|
||||||
|
&'a self,
|
||||||
|
slots: &'a [usize],
|
||||||
|
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
||||||
self.data.nonzero_iter(slots)
|
self.data.nonzero_iter(slots)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -477,7 +480,10 @@ impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix + obicompactvec::ColumnWeig
|
|||||||
impl TypedLayer<PersistentBitMatrix> {
|
impl TypedLayer<PersistentBitMatrix> {
|
||||||
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
|
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
|
||||||
/// delegates to `PersistentBitMatrix::nonzero_iter`.
|
/// delegates to `PersistentBitMatrix::nonzero_iter`.
|
||||||
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
pub fn nonzero_iter<'a>(
|
||||||
|
&'a self,
|
||||||
|
slots: &'a [usize],
|
||||||
|
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
||||||
self.data.nonzero_iter(slots)
|
self.data.nonzero_iter(slots)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -24,14 +24,14 @@ use std::fs;
|
|||||||
use std::io;
|
use std::io;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
|
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
|
||||||
use cacheline_ef::{CachelineEf, CachelineEfVec};
|
use cacheline_ef::{CachelineEf, CachelineEfVec};
|
||||||
use epserde::prelude::*;
|
use epserde::prelude::*;
|
||||||
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
|
use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec};
|
||||||
use obidebruinj::GraphDeBruijn;
|
use obidebruinj::GraphDeBruijn;
|
||||||
use obikindex::layer::IndexMode;
|
use obikindex::layer::IndexMode;
|
||||||
use obikindex::layer::{KmerLayer, TypedLayer};
|
use obikindex::layer::{KmerLayer, TypedLayer};
|
||||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||||
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
|
|
||||||
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
||||||
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
||||||
|
|
||||||
@@ -139,7 +139,9 @@ impl PrivateBuilder for KmerIndex {
|
|||||||
mode: &IndexMode,
|
mode: &IndexMode,
|
||||||
block_bits: u8,
|
block_bits: u8,
|
||||||
) -> Result<usize, SKError> {
|
) -> Result<usize, SKError> {
|
||||||
let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?;
|
let layer0 = self
|
||||||
|
.layer0(i)
|
||||||
|
.map_err(|e| io::Error::other(e.to_string()))?;
|
||||||
let layer0_dir = layer0.dir();
|
let layer0_dir = layer0.dir();
|
||||||
let dedup_path = layer0.dereplicated_superkmers_path();
|
let dedup_path = layer0.dereplicated_superkmers_path();
|
||||||
if !dedup_path.exists() {
|
if !dedup_path.exists() {
|
||||||
@@ -193,15 +195,12 @@ impl PrivateBuilder for KmerIndex {
|
|||||||
let n_kmers = if with_counts {
|
let n_kmers = if with_counts {
|
||||||
let n = write_graph_as_unitigs(g, layer0_dir)
|
let n = write_graph_as_unitigs(g, layer0_dir)
|
||||||
.map_err(|e| io::Error::other(e.to_string()))?;
|
.map_err(|e| io::Error::other(e.to_string()))?;
|
||||||
TypedLayer::<PersistentCompactIntMatrix>::build(
|
TypedLayer::<PersistentIntMatrix>::build(layer0_dir, block_bits, mode, |kmer| {
|
||||||
layer0_dir,
|
match (&mphf1_opt, &counts1_opt) {
|
||||||
block_bits,
|
|
||||||
mode,
|
|
||||||
|kmer| match (&mphf1_opt, &counts1_opt) {
|
|
||||||
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
|
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
|
||||||
_ => 1,
|
_ => 1,
|
||||||
},
|
}
|
||||||
)
|
})
|
||||||
.map_err(|e| io::Error::other(e.to_string()))?;
|
.map_err(|e| io::Error::other(e.to_string()))?;
|
||||||
n
|
n
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ fn presence(mode: MergeMode) -> bool {
|
|||||||
mod matrix_builder_tests {
|
mod matrix_builder_tests {
|
||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
|
||||||
|
|
||||||
use super::{ColBuilder, MatrixBuilder};
|
use super::{ColBuilder, MatrixBuilder};
|
||||||
|
|
||||||
@@ -90,7 +90,7 @@ mod matrix_builder_tests {
|
|||||||
col.close().unwrap();
|
col.close().unwrap();
|
||||||
mb.close().unwrap();
|
mb.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 2);
|
assert_eq!(m.n_cols(), 2);
|
||||||
assert_eq!(&*m.row(0), &[0u32, 7]);
|
assert_eq!(&*m.row(0), &[0u32, 7]);
|
||||||
assert_eq!(&*m.row(1), &[0u32, 42]);
|
assert_eq!(&*m.row(1), &[0u32, 42]);
|
||||||
@@ -422,8 +422,13 @@ pub(crate) fn merge_partition(
|
|||||||
move |data: Pass2Data,
|
move |data: Pass2Data,
|
||||||
push: &PipelineSender<Result<Pass2Data, PipelineError>>,
|
push: &PipelineSender<Result<Pass2Data, PipelineError>>,
|
||||||
delta: &PipelineSender<isize>| {
|
delta: &PipelineSender<isize>| {
|
||||||
if let Pass2Data::SrcLayer((col_offset, src_n, unitigs_path, src_layer, _guard)) =
|
if let Pass2Data::SrcLayer((
|
||||||
data
|
col_offset,
|
||||||
|
src_n,
|
||||||
|
unitigs_path,
|
||||||
|
src_layer,
|
||||||
|
_guard,
|
||||||
|
)) = data
|
||||||
{
|
{
|
||||||
// _guard dropped at end of block, releasing the slot.
|
// _guard dropped at end of block, releasing the slot.
|
||||||
// `src_layer` (MPHF + matrix) was already opened up
|
// `src_layer` (MPHF + matrix) was already opened up
|
||||||
@@ -455,7 +460,9 @@ pub(crate) fn merge_partition(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !batch.is_empty() {
|
if !batch.is_empty() {
|
||||||
push.send(Ok(Pass2Data::RawBatch((col_offset, src_n, src_layer, batch))))
|
push.send(Ok(Pass2Data::RawBatch((
|
||||||
|
col_offset, src_n, src_layer, batch,
|
||||||
|
))))
|
||||||
.ok();
|
.ok();
|
||||||
count += 1;
|
count += 1;
|
||||||
}
|
}
|
||||||
@@ -480,8 +487,9 @@ pub(crate) fn merge_partition(
|
|||||||
// per kmer — matches the underlying matrix's own
|
// per kmer — matches the underlying matrix's own
|
||||||
// column-major layout.
|
// column-major layout.
|
||||||
let slots = src_layer.hash_batch(&kmers);
|
let slots = src_layer.hash_batch(&kmers);
|
||||||
let mut cols: Vec<Vec<u32>> =
|
let mut cols: Vec<Vec<u32>> = (0..src_n)
|
||||||
(0..src_n).map(|_| Vec::with_capacity(kmers.len())).collect();
|
.map(|_| Vec::with_capacity(kmers.len()))
|
||||||
|
.collect();
|
||||||
src_layer.fill_sub_matrix(&slots, &mut cols);
|
src_layer.fill_sub_matrix(&slots, &mut cols);
|
||||||
|
|
||||||
// Membership against dst, grouped by dst layer
|
// Membership against dst, grouped by dst layer
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
|
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
|
||||||
|
use obikidxcache::LayeredStore;
|
||||||
use obikindex::OKIResult;
|
use obikindex::OKIResult;
|
||||||
use obikindex::layer::open_data;
|
use obikindex::layer::open_data;
|
||||||
use obikidxcache::LayeredStore;
|
|
||||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
|
||||||
|
|
||||||
use obikindex::load_meta;
|
|
||||||
use obikindex::KmerIndex;
|
use obikindex::KmerIndex;
|
||||||
|
use obikindex::load_meta;
|
||||||
|
|
||||||
impl KmerIndex {
|
impl KmerIndex {
|
||||||
/// Open all count matrices for partition `part`, one per layer.
|
/// Open all count matrices for partition `part`, one per layer.
|
||||||
/// Layers without a `counts/` directory are skipped.
|
/// Layers without a `counts/` directory are skipped.
|
||||||
pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentCompactIntMatrix>> {
|
pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentIntMatrix>> {
|
||||||
let index_dir = self.index_dir(part);
|
let index_dir = self.index_dir(part);
|
||||||
if !index_dir.exists() {
|
if !index_dir.exists() {
|
||||||
return Ok(LayeredStore::new(vec![]));
|
return Ok(LayeredStore::new(vec![]));
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ use std::io;
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use obicompactvec::{
|
use obicompactvec::{
|
||||||
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix,
|
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix, PersistentIntMatrix, TempBitVec,
|
||||||
PersistentCompactIntMatrix, TempBitVec, TempCompactIntVec,
|
TempCompactIntVec,
|
||||||
};
|
};
|
||||||
use obikindex::layer::{KmerLayer, LayerContent};
|
use obikindex::layer::{KmerLayer, LayerContent};
|
||||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||||
@@ -45,7 +45,9 @@ impl AggOp {
|
|||||||
"sum" => Ok(AggOp::Sum),
|
"sum" => Ok(AggOp::Sum),
|
||||||
"min" => Ok(AggOp::Min),
|
"min" => Ok(AggOp::Min),
|
||||||
"max" => Ok(AggOp::Max),
|
"max" => Ok(AggOp::Max),
|
||||||
other => Err(format!("unknown aggregation operator: {other}; valid: any, all, none, sum, min, max")),
|
other => Err(format!(
|
||||||
|
"unknown aggregation operator: {other}; valid: any, all, none, sum, min, max"
|
||||||
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,7 +77,11 @@ enum AggResult {
|
|||||||
Int(TempCompactIntVec),
|
Int(TempCompactIntVec),
|
||||||
}
|
}
|
||||||
|
|
||||||
fn compute_group(mat: &dyn MatrixGroupOps, spec: &OutputCol, threshold: u32) -> io::Result<AggResult> {
|
fn compute_group(
|
||||||
|
mat: &dyn MatrixGroupOps,
|
||||||
|
spec: &OutputCol,
|
||||||
|
threshold: u32,
|
||||||
|
) -> io::Result<AggResult> {
|
||||||
let g = ColGroup::new(spec.label.clone(), spec.indices.clone());
|
let g = ColGroup::new(spec.label.clone(), spec.indices.clone());
|
||||||
Ok(match spec.op {
|
Ok(match spec.op {
|
||||||
AggOp::Any => AggResult::Bit(mat.partial_group_any(&g, threshold)?),
|
AggOp::Any => AggResult::Bit(mat.partial_group_any(&g, threshold)?),
|
||||||
@@ -145,18 +151,23 @@ pub(crate) fn select_partition(
|
|||||||
|
|
||||||
let group_mat: Box<dyn MatrixGroupOps> = match src_layer.content() {
|
let group_mat: Box<dyn MatrixGroupOps> = match src_layer.content() {
|
||||||
LayerContent::Count => {
|
LayerContent::Count => {
|
||||||
Box::new(PersistentCompactIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
|
Box::new(PersistentIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
|
||||||
}
|
}
|
||||||
LayerContent::Presence => {
|
LayerContent::Presence => {
|
||||||
Box::new(PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
|
Box::new(PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let data_subdir = if output_presence { "presence" } else { "counts" };
|
let data_subdir = if output_presence {
|
||||||
|
"presence"
|
||||||
|
} else {
|
||||||
|
"counts"
|
||||||
|
};
|
||||||
let data_dir = dst_layer_dir.join(data_subdir);
|
let data_dir = dst_layer_dir.join(data_subdir);
|
||||||
fs::create_dir_all(&data_dir).map_err(OKIError::Io)?;
|
fs::create_dir_all(&data_dir).map_err(OKIError::Io)?;
|
||||||
|
|
||||||
let mut builder = MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?;
|
let mut builder =
|
||||||
|
MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?;
|
||||||
for spec in specs {
|
for spec in specs {
|
||||||
let r = compute_group(group_mat.as_ref(), spec, threshold).map_err(OKIError::Io)?;
|
let r = compute_group(group_mat.as_ref(), spec, threshold).map_err(OKIError::Io)?;
|
||||||
add_result(&mut builder, r).map_err(OKIError::Io)?;
|
add_result(&mut builder, r).map_err(OKIError::Io)?;
|
||||||
|
|||||||
Reference in New Issue
Block a user