Rename PersistentCompactIntMatrix to PersistentIntMatrix
Update all matrix type references, imports, and instantiations across the codebase to use the new PersistentIntMatrix name. The change also includes standardizing code formatting, such as multi-line statements and import ordering, without altering any behavioral logic or public API contracts.
This commit is contained in:
@@ -13,20 +13,30 @@ use std::error::Error;
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
use obicompactvec::{PersistentCompactIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder};
|
use obicompactvec::{
|
||||||
|
PersistentIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
|
||||||
|
};
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
fn main() -> Result<(), Box<dyn Error>> {
|
||||||
let layer_dir = std::env::args().nth(1).expect("usage: compare_sparse_dense_count <layer_dir>");
|
let layer_dir = std::env::args()
|
||||||
|
.nth(1)
|
||||||
|
.expect("usage: compare_sparse_dense_count <layer_dir>");
|
||||||
let layer_dir = Path::new(&layer_dir);
|
let layer_dir = Path::new(&layer_dir);
|
||||||
|
|
||||||
let t0 = Instant::now();
|
let t0 = Instant::now();
|
||||||
let dense = PersistentCompactIntMatrix::open(layer_dir)?;
|
let dense = PersistentIntMatrix::open(layer_dir)?;
|
||||||
println!("dense ouverte en {:?} ({} lignes x {} colonnes)", t0.elapsed(), dense.n(), dense.n_cols());
|
println!(
|
||||||
|
"dense ouverte en {:?} ({} lignes x {} colonnes)",
|
||||||
|
t0.elapsed(),
|
||||||
|
dense.n(),
|
||||||
|
dense.n_cols()
|
||||||
|
);
|
||||||
|
|
||||||
let out_dir = tempfile::tempdir()?;
|
let out_dir = tempfile::tempdir()?;
|
||||||
|
|
||||||
let t0 = Instant::now();
|
let t0 = Instant::now();
|
||||||
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?.finish()?;
|
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?
|
||||||
|
.finish()?;
|
||||||
println!("build_from_dense: {:?}", t0.elapsed());
|
println!("build_from_dense: {:?}", t0.elapsed());
|
||||||
|
|
||||||
compare(&dense, &sparse)?;
|
compare(&dense, &sparse)?;
|
||||||
@@ -36,7 +46,10 @@ fn main() -> Result<(), Box<dyn Error>> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactIntMatrix) -> Result<(), Box<dyn Error>> {
|
fn compare(
|
||||||
|
dense: &PersistentIntMatrix,
|
||||||
|
sparse: &PersistentSparseCompactIntMatrix,
|
||||||
|
) -> Result<(), Box<dyn Error>> {
|
||||||
let n = dense.n();
|
let n = dense.n();
|
||||||
let n_cols = dense.n_cols();
|
let n_cols = dense.n_cols();
|
||||||
assert_eq!(n, sparse.n(), "n mismatch");
|
assert_eq!(n, sparse.n(), "n mismatch");
|
||||||
@@ -63,7 +76,10 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
println!("comparaison case-a-case: {:?} ({total_cells} cellules)", t0.elapsed());
|
println!(
|
||||||
|
"comparaison case-a-case: {:?} ({total_cells} cellules)",
|
||||||
|
t0.elapsed()
|
||||||
|
);
|
||||||
|
|
||||||
if let Some((slot, col, d, s)) = first_mismatch {
|
if let Some((slot, col, d, s)) = first_mismatch {
|
||||||
eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}");
|
eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}");
|
||||||
@@ -78,7 +94,7 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI
|
|||||||
/// for what's actually stored) — sparsity, singleton-row fraction, and the
|
/// for what's actually stored) — sparsity, singleton-row fraction, and the
|
||||||
/// value-magnitude buckets this crate's overflow encoding is tuned around
|
/// value-magnitude buckets this crate's overflow encoding is tuned around
|
||||||
/// (< 127, < 255, >= 255).
|
/// (< 127, < 255, >= 255).
|
||||||
fn content_stats(dense: &PersistentCompactIntMatrix) {
|
fn content_stats(dense: &PersistentIntMatrix) {
|
||||||
let n = dense.n();
|
let n = dense.n();
|
||||||
let n_cols = dense.n_cols();
|
let n_cols = dense.n_cols();
|
||||||
let total_cells = n as u64 * n_cols as u64;
|
let total_cells = n as u64 * n_cols as u64;
|
||||||
@@ -128,7 +144,10 @@ fn content_stats(dense: &PersistentCompactIntMatrix) {
|
|||||||
100.0 * singleton_rows as f64 / n as f64
|
100.0 * singleton_rows as f64 / n as f64
|
||||||
);
|
);
|
||||||
if nonzero_cells > 0 {
|
if nonzero_cells > 0 {
|
||||||
println!("valeur moyenne (non-nulles): {:.2}", sum as f64 / nonzero_cells as f64);
|
println!(
|
||||||
|
"valeur moyenne (non-nulles): {:.2}",
|
||||||
|
sum as f64 / nonzero_cells as f64
|
||||||
|
);
|
||||||
println!("valeur max: {max_value}");
|
println!("valeur max: {max_value}");
|
||||||
println!(
|
println!(
|
||||||
"valeurs < 127: {under_127} ({:.3}% des non-nulles)",
|
"valeurs < 127: {under_127} ({:.3}% des non-nulles)",
|
||||||
@@ -162,7 +181,7 @@ fn dir_size(dir: &Path) -> std::io::Result<u64> {
|
|||||||
fn compaction_stats(
|
fn compaction_stats(
|
||||||
dense_layer_dir: &Path,
|
dense_layer_dir: &Path,
|
||||||
sparse_dir: &Path,
|
sparse_dir: &Path,
|
||||||
dense: &PersistentCompactIntMatrix,
|
dense: &PersistentIntMatrix,
|
||||||
_sparse: &PersistentSparseCompactIntMatrix,
|
_sparse: &PersistentSparseCompactIntMatrix,
|
||||||
) -> Result<(), Box<dyn Error>> {
|
) -> Result<(), Box<dyn Error>> {
|
||||||
let dense_size = dir_size(&dense_layer_dir.join("counts"))?;
|
let dense_size = dir_size(&dense_layer_dir.join("counts"))?;
|
||||||
|
|||||||
@@ -5,17 +5,19 @@ use std::path::{Path, PathBuf};
|
|||||||
use memmap2::Mmap;
|
use memmap2::Mmap;
|
||||||
use ndarray::{Array1, Array2};
|
use ndarray::{Array1, Array2};
|
||||||
|
|
||||||
use crate::bitmatrix::{fill_row_generic, par_col_reduce, pairwise2_matrix, pairwise_matrix, row_generic};
|
use crate::bitmatrix::{
|
||||||
|
fill_row_generic, pairwise_matrix, pairwise2_matrix, par_col_reduce, row_generic,
|
||||||
|
};
|
||||||
use crate::builder::PersistentCompactIntVecBuilder;
|
use crate::builder::PersistentCompactIntVecBuilder;
|
||||||
use crate::colgroup::{chunked_presence_count, ColGroup, MatrixGroupOps};
|
use crate::colgroup::{ColGroup, MatrixGroupOps, chunked_presence_count};
|
||||||
use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE};
|
use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE};
|
||||||
use crate::meta::MatrixMeta;
|
use crate::meta::MatrixMeta;
|
||||||
use crate::reader::PersistentCompactIntVec;
|
use crate::reader::PersistentCompactIntVec;
|
||||||
use crate::sparse_intmatrix::{is_present as sparse_is_present, PersistentSparseCompactIntMatrix};
|
use crate::sparse_intmatrix::{PersistentSparseCompactIntMatrix, is_present as sparse_is_present};
|
||||||
use crate::storage_kind::StorageKind;
|
use crate::storage_kind::StorageKind;
|
||||||
use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
|
use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
|
||||||
use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
|
use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
|
||||||
use crate::views::{nonzero_triples, IntSliceView};
|
use crate::views::{IntSliceView, nonzero_triples};
|
||||||
|
|
||||||
pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf {
|
pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf {
|
||||||
dir.join(format!("col_{col:06}.pciv"))
|
dir.join(format!("col_{col:06}.pciv"))
|
||||||
@@ -38,11 +40,17 @@ impl ColumnarCompactIntMatrix {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn n(&self) -> usize { self.n }
|
pub(crate) fn n(&self) -> usize {
|
||||||
|
self.n
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn n_cols(&self) -> usize { self.cols.len() }
|
pub(crate) fn n_cols(&self) -> usize {
|
||||||
|
self.cols.len()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec { &self.cols[c] }
|
pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec {
|
||||||
|
&self.cols[c]
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn row(&self, slot: usize) -> Box<[u32]> {
|
pub(crate) fn row(&self, slot: usize) -> Box<[u32]> {
|
||||||
@@ -63,34 +71,64 @@ impl ColumnarCompactIntMatrix {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
||||||
pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_bray_dist(self.col(j)))
|
pairwise_matrix(self.n_cols(), |i, j| {
|
||||||
|
self.col(i).partial_bray_dist(self.col(j))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
|
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_euclidean_dist(self.col(j)))
|
pairwise_matrix(self.n_cols(), |i, j| {
|
||||||
|
self.col(i).partial_euclidean_dist(self.col(j))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
|
pub(crate) fn partial_threshold_jaccard_dist_matrix(
|
||||||
pairwise2_matrix(self.n_cols(), |i, j| self.col(i).partial_threshold_jaccard_dist(self.col(j), threshold))
|
&self,
|
||||||
|
threshold: u32,
|
||||||
|
) -> (Array2<u64>, Array2<u64>) {
|
||||||
|
pairwise2_matrix(self.n_cols(), |i, j| {
|
||||||
|
self.col(i)
|
||||||
|
.partial_threshold_jaccard_dist(self.col(j), threshold)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols(), |i, j| {
|
pairwise_matrix(self.n_cols(), |i, j| {
|
||||||
self.col(i).partial_relfreq_bray_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col(i).partial_relfreq_bray_dist(
|
||||||
|
self.col(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_relfreq_euclidean_dist_matrix(
|
||||||
|
&self,
|
||||||
|
col_sums: &Array1<u64>,
|
||||||
|
) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols(), |i, j| {
|
pairwise_matrix(self.n_cols(), |i, j| {
|
||||||
self.col(i).partial_relfreq_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col(i).partial_relfreq_euclidean_dist(
|
||||||
|
self.col(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_hellinger_euclidean_dist_matrix(
|
||||||
|
&self,
|
||||||
|
col_sums: &Array1<u64>,
|
||||||
|
) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols(), |i, j| {
|
pairwise_matrix(self.n_cols(), |i, j| {
|
||||||
self.col(i).partial_hellinger_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col(i).partial_hellinger_euclidean_dist(
|
||||||
|
self.col(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> {
|
pub(crate) fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> {
|
||||||
let mut meta = MatrixMeta::load(dir)?;
|
let mut meta = MatrixMeta::load(dir)?;
|
||||||
let mut b = PersistentCompactIntVecBuilder::new(meta.n, &col_path(dir, meta.n_cols))?;
|
let mut b = PersistentCompactIntVecBuilder::new(meta.n, &col_path(dir, meta.n_cols))?;
|
||||||
for slot in 0..meta.n { b.set(slot, value_of(slot)); }
|
for slot in 0..meta.n {
|
||||||
|
b.set(slot, value_of(slot));
|
||||||
|
}
|
||||||
b.close()?;
|
b.close()?;
|
||||||
meta.n_cols += 1;
|
meta.n_cols += 1;
|
||||||
meta.save(dir)
|
meta.save(dir)
|
||||||
@@ -119,7 +157,10 @@ impl PackedCompactIntMatrix {
|
|||||||
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
||||||
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
|
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
|
||||||
if mmap.len() < PCMX_HEADER {
|
if mmap.len() < PCMX_HEADER {
|
||||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "PCMX file too short"));
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"PCMX file too short",
|
||||||
|
));
|
||||||
}
|
}
|
||||||
if &mmap[0..4] != &PCMX_MAGIC {
|
if &mmap[0..4] != &PCMX_MAGIC {
|
||||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PCMX magic"));
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PCMX magic"));
|
||||||
@@ -130,32 +171,51 @@ impl PackedCompactIntMatrix {
|
|||||||
let mut columns = Vec::with_capacity(n_cols);
|
let mut columns = Vec::with_capacity(n_cols);
|
||||||
for c in 0..n_cols {
|
for c in 0..n_cols {
|
||||||
let off_pos = PCMX_HEADER + c * 8;
|
let off_pos = PCMX_HEADER + c * 8;
|
||||||
let col_base = u64::from_le_bytes(mmap[off_pos..off_pos+8].try_into().unwrap()) as usize;
|
let col_base =
|
||||||
let n_ov = u64::from_le_bytes(mmap[col_base+16..col_base+24].try_into().unwrap()) as usize;
|
u64::from_le_bytes(mmap[off_pos..off_pos + 8].try_into().unwrap()) as usize;
|
||||||
let n_pciv = u64::from_le_bytes(mmap[col_base+8..col_base+16].try_into().unwrap()) as usize;
|
let n_ov =
|
||||||
|
u64::from_le_bytes(mmap[col_base + 16..col_base + 24].try_into().unwrap()) as usize;
|
||||||
|
let n_pciv =
|
||||||
|
u64::from_le_bytes(mmap[col_base + 8..col_base + 16].try_into().unwrap()) as usize;
|
||||||
let primary_start = col_base + HEADER_SIZE;
|
let primary_start = col_base + HEADER_SIZE;
|
||||||
let data_offset = primary_start + n_pciv;
|
let data_offset = primary_start + n_pciv;
|
||||||
columns.push(ColInfo { primary_start, data_offset, n_overflow: n_ov });
|
columns.push(ColInfo {
|
||||||
|
primary_start,
|
||||||
|
data_offset,
|
||||||
|
n_overflow: n_ov,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
Ok(Self { mmap, n_rows, n_cols, columns })
|
Ok(Self {
|
||||||
|
mmap,
|
||||||
|
n_rows,
|
||||||
|
n_cols,
|
||||||
|
columns,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> {
|
pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> {
|
||||||
let ci = &self.columns[c];
|
let ci = &self.columns[c];
|
||||||
let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
|
let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
|
||||||
let overflow_raw = &self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE];
|
let overflow_raw =
|
||||||
|
&self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE];
|
||||||
IntSliceView::new(primary, overflow_raw, ci.n_overflow, self.n_rows)
|
IntSliceView::new(primary, overflow_raw, ci.n_overflow, self.n_rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentCompactIntVecBuilder> {
|
pub(crate) fn col_persist(
|
||||||
|
&self,
|
||||||
|
c: usize,
|
||||||
|
path: &Path,
|
||||||
|
) -> io::Result<PersistentCompactIntVecBuilder> {
|
||||||
let view = self.col_view(c);
|
let view = self.col_view(c);
|
||||||
let overflow: std::collections::HashMap<usize, u32> = view.overflow_entries().collect();
|
let overflow: std::collections::HashMap<usize, u32> = view.overflow_entries().collect();
|
||||||
PersistentCompactIntVecBuilder::from_raw_primary(view.primary_bytes(), overflow, path)
|
PersistentCompactIntVecBuilder::from_raw_primary(view.primary_bytes(), overflow, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn get(&self, col: usize, slot: usize) -> u32 { self.col_view(col).get(slot) }
|
pub(crate) fn get(&self, col: usize, slot: usize) -> u32 {
|
||||||
|
self.col_view(col).get(slot)
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
|
pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
|
||||||
@@ -180,27 +240,55 @@ impl PackedCompactIntMatrix {
|
|||||||
// via `PersistentCompactIntVec`. No formula is re-derived here.
|
// via `PersistentCompactIntVec`. No formula is re-derived here.
|
||||||
|
|
||||||
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
||||||
pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_bray_dist(self.col_view(j)))
|
pairwise_matrix(self.n_cols, |i, j| {
|
||||||
|
self.col_view(i).partial_bray_dist(self.col_view(j))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
|
pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_euclidean_dist(self.col_view(j)))
|
pairwise_matrix(self.n_cols, |i, j| {
|
||||||
|
self.col_view(i).partial_euclidean_dist(self.col_view(j))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, t: u32) -> (Array2<u64>, Array2<u64>) {
|
pub(crate) fn partial_threshold_jaccard_dist_matrix(
|
||||||
pairwise2_matrix(self.n_cols, |i, j| self.col_view(i).partial_threshold_jaccard_dist(self.col_view(j), t))
|
&self,
|
||||||
|
t: u32,
|
||||||
|
) -> (Array2<u64>, Array2<u64>) {
|
||||||
|
pairwise2_matrix(self.n_cols, |i, j| {
|
||||||
|
self.col_view(i)
|
||||||
|
.partial_threshold_jaccard_dist(self.col_view(j), t)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols, |i, j| {
|
pairwise_matrix(self.n_cols, |i, j| {
|
||||||
self.col_view(i).partial_relfreq_bray_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col_view(i).partial_relfreq_bray_dist(
|
||||||
|
self.col_view(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_relfreq_euclidean_dist_matrix(
|
||||||
|
&self,
|
||||||
|
col_sums: &Array1<u64>,
|
||||||
|
) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols, |i, j| {
|
pairwise_matrix(self.n_cols, |i, j| {
|
||||||
self.col_view(i).partial_relfreq_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col_view(i).partial_relfreq_euclidean_dist(
|
||||||
|
self.col_view(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
|
pub(crate) fn partial_hellinger_euclidean_dist_matrix(
|
||||||
|
&self,
|
||||||
|
col_sums: &Array1<u64>,
|
||||||
|
) -> Array2<f64> {
|
||||||
pairwise_matrix(self.n_cols, |i, j| {
|
pairwise_matrix(self.n_cols, |i, j| {
|
||||||
self.col_view(i).partial_hellinger_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64)
|
self.col_view(i).partial_hellinger_euclidean_dist(
|
||||||
|
self.col_view(j),
|
||||||
|
col_sums[i] as f64,
|
||||||
|
col_sums[j] as f64,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -238,7 +326,9 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
|||||||
// the columnar files are newer and must be (re-)packed, overwriting the
|
// the columnar files are newer and must be (re-)packed, overwriting the
|
||||||
// stale one — never silently discarded as "leftover cleanup".
|
// stale one — never silently discarded as "leftover cleanup".
|
||||||
if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
|
if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
|
||||||
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
|
for c in 0..meta.n_cols {
|
||||||
|
let _ = fs::remove_file(col_path(dir, c));
|
||||||
|
}
|
||||||
let _ = fs::remove_file(dir.join("meta.json"));
|
let _ = fs::remove_file(dir.join("meta.json"));
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
@@ -250,32 +340,41 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
|||||||
let header_size = (PCMX_HEADER + n_cols * 8) as u64;
|
let header_size = (PCMX_HEADER + n_cols * 8) as u64;
|
||||||
let mut col_offset = header_size;
|
let mut col_offset = header_size;
|
||||||
let mut offsets = Vec::with_capacity(n_cols);
|
let mut offsets = Vec::with_capacity(n_cols);
|
||||||
for &size in &col_sizes { offsets.push(col_offset); col_offset += size; }
|
for &size in &col_sizes {
|
||||||
|
offsets.push(col_offset);
|
||||||
|
col_offset += size;
|
||||||
|
}
|
||||||
let tmp_path = dir.join("matrix.pcmx.tmp");
|
let tmp_path = dir.join("matrix.pcmx.tmp");
|
||||||
let mut out = BufWriter::new(File::create(&tmp_path)?);
|
let mut out = BufWriter::new(File::create(&tmp_path)?);
|
||||||
out.write_all(&PCMX_MAGIC)?;
|
out.write_all(&PCMX_MAGIC)?;
|
||||||
out.write_all(&[0u8; 4])?;
|
out.write_all(&[0u8; 4])?;
|
||||||
out.write_all(&(meta.n as u64).to_le_bytes())?;
|
out.write_all(&(meta.n as u64).to_le_bytes())?;
|
||||||
out.write_all(&(n_cols as u64).to_le_bytes())?;
|
out.write_all(&(n_cols as u64).to_le_bytes())?;
|
||||||
for &off in &offsets { out.write_all(&off.to_le_bytes())?; }
|
for &off in &offsets {
|
||||||
for c in 0..n_cols { io::copy(&mut File::open(col_path(dir, c))?, &mut out)?; }
|
out.write_all(&off.to_le_bytes())?;
|
||||||
|
}
|
||||||
|
for c in 0..n_cols {
|
||||||
|
io::copy(&mut File::open(col_path(dir, c))?, &mut out)?;
|
||||||
|
}
|
||||||
out.flush()?;
|
out.flush()?;
|
||||||
drop(out);
|
drop(out);
|
||||||
fs::rename(&tmp_path, &packed_path)?;
|
fs::rename(&tmp_path, &packed_path)?;
|
||||||
for c in 0..n_cols { fs::remove_file(col_path(dir, c))?; }
|
for c in 0..n_cols {
|
||||||
|
fs::remove_file(col_path(dir, c))?;
|
||||||
|
}
|
||||||
fs::remove_file(dir.join("meta.json"))?;
|
fs::remove_file(dir.join("meta.json"))?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── PersistentCompactIntMatrix — public enum ──────────────────────────────────
|
// ── PersistentCompactIntMatrix — public enum ──────────────────────────────────
|
||||||
|
|
||||||
pub enum PersistentCompactIntMatrix {
|
pub enum PersistentIntMatrix {
|
||||||
Columnar(ColumnarCompactIntMatrix),
|
Columnar(ColumnarCompactIntMatrix),
|
||||||
Packed(PackedCompactIntMatrix),
|
Packed(PackedCompactIntMatrix),
|
||||||
Sparse(PersistentSparseCompactIntMatrix),
|
Sparse(PersistentSparseCompactIntMatrix),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PersistentCompactIntMatrix {
|
impl PersistentIntMatrix {
|
||||||
/// Checks (in order): `counts/matrix.pcmx` → Packed; `counts/meta.json`
|
/// Checks (in order): `counts/matrix.pcmx` → Packed; `counts/meta.json`
|
||||||
/// → Columnar; `counts/singleton_values.pciv` → Sparse. Mirrors
|
/// → Columnar; `counts/singleton_values.pciv` → Sparse. Mirrors
|
||||||
/// `PersistentBitMatrix::open`'s own Packed → Columnar → Sparse
|
/// `PersistentBitMatrix::open`'s own Packed → Columnar → Sparse
|
||||||
@@ -283,17 +382,24 @@ impl PersistentCompactIntMatrix {
|
|||||||
pub fn open(layer_dir: &Path) -> io::Result<Self> {
|
pub fn open(layer_dir: &Path) -> io::Result<Self> {
|
||||||
let counts_dir = layer_dir.join("counts");
|
let counts_dir = layer_dir.join("counts");
|
||||||
if counts_dir.join("matrix.pcmx").exists() {
|
if counts_dir.join("matrix.pcmx").exists() {
|
||||||
return Ok(Self::Packed(PackedCompactIntMatrix::open(&counts_dir.join("matrix.pcmx"))?));
|
return Ok(Self::Packed(PackedCompactIntMatrix::open(
|
||||||
|
&counts_dir.join("matrix.pcmx"),
|
||||||
|
)?));
|
||||||
}
|
}
|
||||||
if MatrixMeta::load(&counts_dir).is_ok() {
|
if MatrixMeta::load(&counts_dir).is_ok() {
|
||||||
return Ok(Self::Columnar(ColumnarCompactIntMatrix::open(&counts_dir)?));
|
return Ok(Self::Columnar(ColumnarCompactIntMatrix::open(&counts_dir)?));
|
||||||
}
|
}
|
||||||
if sparse_is_present(&counts_dir) {
|
if sparse_is_present(&counts_dir) {
|
||||||
return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open(&counts_dir)?));
|
return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open(
|
||||||
|
&counts_dir,
|
||||||
|
)?));
|
||||||
}
|
}
|
||||||
Err(io::Error::new(
|
Err(io::Error::new(
|
||||||
io::ErrorKind::NotFound,
|
io::ErrorKind::NotFound,
|
||||||
format!("no count matrix found in {} — run 'obikmer upgrade'", layer_dir.display()),
|
format!(
|
||||||
|
"no count matrix found in {} — run 'obikmer upgrade'",
|
||||||
|
layer_dir.display()
|
||||||
|
),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -330,11 +436,19 @@ impl PersistentCompactIntMatrix {
|
|||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n(&self) -> usize {
|
pub fn n(&self) -> usize {
|
||||||
match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows, Self::Sparse(m) => m.n() }
|
match self {
|
||||||
|
Self::Columnar(m) => m.n(),
|
||||||
|
Self::Packed(m) => m.n_rows,
|
||||||
|
Self::Sparse(m) => m.n(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n_cols(&self) -> usize {
|
pub fn n_cols(&self) -> usize {
|
||||||
match self { Self::Columnar(m) => m.n_cols(), Self::Packed(m) => m.n_cols, Self::Sparse(m) => m.n_cols() }
|
match self {
|
||||||
|
Self::Columnar(m) => m.n_cols(),
|
||||||
|
Self::Packed(m) => m.n_cols,
|
||||||
|
Self::Sparse(m) => m.n_cols(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
@@ -350,7 +464,9 @@ impl PersistentCompactIntMatrix {
|
|||||||
match self {
|
match self {
|
||||||
Self::Columnar(m) => m.col(c).view(),
|
Self::Columnar(m) => m.col(c).view(),
|
||||||
Self::Packed(m) => m.col_view(c),
|
Self::Packed(m) => m.col_view(c),
|
||||||
Self::Sparse(_) => panic!("col_view() not available on Sparse PersistentCompactIntMatrix"),
|
Self::Sparse(_) => {
|
||||||
|
panic!("col_view() not available on Sparse PersistentCompactIntMatrix")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -358,14 +474,20 @@ impl PersistentCompactIntMatrix {
|
|||||||
match self {
|
match self {
|
||||||
Self::Columnar(m) => PersistentCompactIntVecBuilder::build_from(m.col(c), path),
|
Self::Columnar(m) => PersistentCompactIntVecBuilder::build_from(m.col(c), path),
|
||||||
Self::Packed(m) => m.col_persist(c, path),
|
Self::Packed(m) => m.col_persist(c, path),
|
||||||
Self::Sparse(_) => Err(io::Error::new(io::ErrorKind::Unsupported,
|
Self::Sparse(_) => Err(io::Error::new(
|
||||||
"col_persist not available on Sparse PersistentCompactIntMatrix")),
|
io::ErrorKind::Unsupported,
|
||||||
|
"col_persist not available on Sparse PersistentCompactIntMatrix",
|
||||||
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn row(&self, slot: usize) -> Box<[u32]> {
|
pub fn row(&self, slot: usize) -> Box<[u32]> {
|
||||||
match self { Self::Columnar(m) => m.row(slot), Self::Packed(m) => m.row(slot), Self::Sparse(m) => m.row(slot) }
|
match self {
|
||||||
|
Self::Columnar(m) => m.row(slot),
|
||||||
|
Self::Packed(m) => m.row(slot),
|
||||||
|
Self::Sparse(m) => m.row(slot),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
|
pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
|
||||||
@@ -418,18 +540,28 @@ impl PersistentCompactIntMatrix {
|
|||||||
/// `Columnar`/`Packed` reuse the shared sorted-slot batching in
|
/// `Columnar`/`Packed` reuse the shared sorted-slot batching in
|
||||||
/// `views::nonzero_triples`. Boxed, not `impl Iterator`, because the
|
/// `views::nonzero_triples`. Boxed, not `impl Iterator`, because the
|
||||||
/// match arms are genuinely different concrete types.
|
/// match arms are genuinely different concrete types.
|
||||||
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
pub fn nonzero_iter<'a>(
|
||||||
|
&'a self,
|
||||||
|
slots: &'a [usize],
|
||||||
|
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
||||||
match self {
|
match self {
|
||||||
Self::Sparse(m) => Box::new(m.nonzero_iter(slots)),
|
Self::Sparse(m) => Box::new(m.nonzero_iter(slots)),
|
||||||
Self::Columnar(_) | Self::Packed(_) => {
|
Self::Columnar(_) | Self::Packed(_) => Box::new(nonzero_triples(
|
||||||
Box::new(nonzero_triples(slots, self.n_cols(), |c| self.col_view(c), false))
|
slots,
|
||||||
}
|
self.n_cols(),
|
||||||
|
|c| self.col_view(c),
|
||||||
|
false,
|
||||||
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn sum(&self) -> Array1<u64> {
|
pub fn sum(&self) -> Array1<u64> {
|
||||||
match self { Self::Columnar(m) => m.sum(), Self::Packed(m) => m.sum(), Self::Sparse(m) => m.sum() }
|
match self {
|
||||||
|
Self::Columnar(m) => m.sum(),
|
||||||
|
Self::Packed(m) => m.sum(),
|
||||||
|
Self::Sparse(m) => m.sum(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn count_nonzero(&self) -> Array1<u64> {
|
pub fn count_nonzero(&self) -> Array1<u64> {
|
||||||
@@ -456,7 +588,10 @@ impl PersistentCompactIntMatrix {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
|
pub fn partial_threshold_jaccard_dist_matrix(
|
||||||
|
&self,
|
||||||
|
threshold: u32,
|
||||||
|
) -> (Array2<u64>, Array2<u64>) {
|
||||||
match self {
|
match self {
|
||||||
Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
|
Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
|
||||||
Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
|
Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold),
|
||||||
@@ -497,26 +632,42 @@ impl PersistentCompactIntMatrix {
|
|||||||
|
|
||||||
use crate::traits::{ColumnWeights, CountPartials};
|
use crate::traits::{ColumnWeights, CountPartials};
|
||||||
|
|
||||||
impl ColumnWeights for PersistentCompactIntMatrix {
|
impl ColumnWeights for PersistentIntMatrix {
|
||||||
#[inline]
|
#[inline]
|
||||||
fn col_weights(&self) -> Array1<u64> { self.sum() }
|
fn col_weights(&self) -> Array1<u64> {
|
||||||
|
self.sum()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() }
|
fn partial_kmer_counts(&self) -> Array1<u64> {
|
||||||
|
self.count_nonzero()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CountPartials for PersistentCompactIntMatrix {
|
impl CountPartials for PersistentIntMatrix {
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_bray(&self) -> Array2<u64> { self.partial_bray_dist_matrix() }
|
fn partial_bray(&self) -> Array2<u64> {
|
||||||
|
self.partial_bray_dist_matrix()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_euclidean(&self) -> Array2<f64> { self.partial_euclidean_dist_matrix() }
|
fn partial_euclidean(&self) -> Array2<f64> {
|
||||||
|
self.partial_euclidean_dist_matrix()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) { self.partial_threshold_jaccard_dist_matrix(t) }
|
fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) {
|
||||||
|
self.partial_threshold_jaccard_dist_matrix(t)
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_bray_dist_matrix(g) }
|
fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> {
|
||||||
|
self.partial_relfreq_bray_dist_matrix(g)
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_euclidean_dist_matrix(g) }
|
fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> {
|
||||||
|
self.partial_relfreq_euclidean_dist_matrix(g)
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_hellinger_euclidean_dist_matrix(g) }
|
fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> {
|
||||||
|
self.partial_hellinger_euclidean_dist_matrix(g)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Builder ───────────────────────────────────────────────────────────────────
|
// ── Builder ───────────────────────────────────────────────────────────────────
|
||||||
@@ -530,7 +681,11 @@ pub struct PersistentCompactIntMatrixBuilder {
|
|||||||
impl PersistentCompactIntMatrixBuilder {
|
impl PersistentCompactIntMatrixBuilder {
|
||||||
pub fn new(n: usize, dir: &Path) -> io::Result<Self> {
|
pub fn new(n: usize, dir: &Path) -> io::Result<Self> {
|
||||||
fs::create_dir_all(dir)?;
|
fs::create_dir_all(dir)?;
|
||||||
Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
|
Ok(Self {
|
||||||
|
dir: dir.to_path_buf(),
|
||||||
|
n,
|
||||||
|
n_cols: 0,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resume appending columns to a Columnar matrix directory already
|
/// Resume appending columns to a Columnar matrix directory already
|
||||||
@@ -541,13 +696,21 @@ impl PersistentCompactIntMatrixBuilder {
|
|||||||
/// on-disk column naming or `MatrixMeta` themselves.
|
/// on-disk column naming or `MatrixMeta` themselves.
|
||||||
pub fn resume(dir: &Path) -> io::Result<Self> {
|
pub fn resume(dir: &Path) -> io::Result<Self> {
|
||||||
let meta = MatrixMeta::load(dir)?;
|
let meta = MatrixMeta::load(dir)?;
|
||||||
Ok(Self { dir: dir.to_path_buf(), n: meta.n, n_cols: meta.n_cols })
|
Ok(Self {
|
||||||
|
dir: dir.to_path_buf(),
|
||||||
|
n: meta.n,
|
||||||
|
n_cols: meta.n_cols,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n(&self) -> usize { self.n }
|
pub fn n(&self) -> usize {
|
||||||
|
self.n
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n_cols(&self) -> usize { self.n_cols }
|
pub fn n_cols(&self) -> usize {
|
||||||
|
self.n_cols
|
||||||
|
}
|
||||||
pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> {
|
pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> {
|
||||||
let path = col_path(&self.dir, self.n_cols);
|
let path = col_path(&self.dir, self.n_cols);
|
||||||
self.n_cols += 1;
|
self.n_cols += 1;
|
||||||
@@ -569,14 +732,22 @@ impl PersistentCompactIntMatrixBuilder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn close(self) -> io::Result<()> {
|
pub fn close(self) -> io::Result<()> {
|
||||||
MatrixMeta { n: self.n, n_cols: self.n_cols }.save(&self.dir)
|
MatrixMeta {
|
||||||
|
n: self.n,
|
||||||
|
n_cols: self.n_cols,
|
||||||
|
}
|
||||||
|
.save(&self.dir)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── MatrixGroupOps ────────────────────────────────────────────────────────────
|
// ── MatrixGroupOps ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
impl MatrixGroupOps for PersistentCompactIntMatrix {
|
impl MatrixGroupOps for PersistentIntMatrix {
|
||||||
fn partial_group_presence_count(&self, g: &ColGroup, threshold: u32) -> io::Result<TempCompactIntVec> {
|
fn partial_group_presence_count(
|
||||||
|
&self,
|
||||||
|
g: &ColGroup,
|
||||||
|
threshold: u32,
|
||||||
|
) -> io::Result<TempCompactIntVec> {
|
||||||
chunked_presence_count(self.n(), &g.indices, |b, c| {
|
chunked_presence_count(self.n(), &g.indices, |b, c| {
|
||||||
b.inc_predicate_fast(self.col_view(c), |v| v >= threshold)
|
b.inc_predicate_fast(self.col_view(c), |v| v >= threshold)
|
||||||
})
|
})
|
||||||
@@ -585,7 +756,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
|
|||||||
fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
|
fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
|
||||||
let n = self.n();
|
let n = self.n();
|
||||||
let mut result = TempCompactIntVecBuilder::new(n)?;
|
let mut result = TempCompactIntVecBuilder::new(n)?;
|
||||||
for &c in &g.indices { result.add(self.col_view(c)); }
|
for &c in &g.indices {
|
||||||
|
result.add(self.col_view(c));
|
||||||
|
}
|
||||||
result.freeze()
|
result.freeze()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -603,7 +776,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
|
|||||||
let mut result = TempCompactIntVecBuilder::new(n)?;
|
let mut result = TempCompactIntVecBuilder::new(n)?;
|
||||||
if let Some((&first, rest)) = g.indices.split_first() {
|
if let Some((&first, rest)) = g.indices.split_first() {
|
||||||
result.add(self.col_view(first));
|
result.add(self.col_view(first));
|
||||||
for &c in rest { result.min(self.col_view(c)); }
|
for &c in rest {
|
||||||
|
result.min(self.col_view(c));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
result.freeze()
|
result.freeze()
|
||||||
}
|
}
|
||||||
@@ -611,7 +786,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix {
|
|||||||
fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
|
fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
|
||||||
let n = self.n();
|
let n = self.n();
|
||||||
let mut result = TempCompactIntVecBuilder::new(n)?;
|
let mut result = TempCompactIntVecBuilder::new(n)?;
|
||||||
for &c in &g.indices { result.max(self.col_view(c)); }
|
for &c in &g.indices {
|
||||||
|
result.max(self.col_view(c));
|
||||||
|
}
|
||||||
result.freeze()
|
result.freeze()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,41 +1,50 @@
|
|||||||
mod bitvec;
|
|
||||||
mod bitmatrix;
|
mod bitmatrix;
|
||||||
|
mod bitvec;
|
||||||
mod builder;
|
mod builder;
|
||||||
mod colgroup;
|
mod colgroup;
|
||||||
mod eliasfano;
|
mod eliasfano;
|
||||||
mod fixedintvec;
|
mod fixedintvec;
|
||||||
mod format;
|
mod format;
|
||||||
mod rankselect;
|
|
||||||
mod intmatrix;
|
mod intmatrix;
|
||||||
mod layer_meta;
|
mod layer_meta;
|
||||||
mod matrix_builder;
|
mod matrix_builder;
|
||||||
mod meta;
|
mod meta;
|
||||||
mod mmap_file;
|
mod mmap_file;
|
||||||
|
mod rankselect;
|
||||||
mod reader;
|
mod reader;
|
||||||
mod sparse_intmatrix;
|
mod sparse_intmatrix;
|
||||||
mod storage_kind;
|
mod storage_kind;
|
||||||
mod tempbitvec;
|
mod tempbitvec;
|
||||||
mod tempintvec;
|
mod tempintvec;
|
||||||
mod views;
|
|
||||||
pub mod traits;
|
pub mod traits;
|
||||||
|
mod views;
|
||||||
|
|
||||||
|
pub use bitmatrix::{
|
||||||
|
PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix,
|
||||||
|
PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix,
|
||||||
|
};
|
||||||
pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
|
pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
|
||||||
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
|
|
||||||
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
|
|
||||||
pub use eliasfano::{EliasFano, EliasFanoBuilder};
|
|
||||||
pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix};
|
|
||||||
pub use builder::PersistentCompactIntVecBuilder;
|
pub use builder::PersistentCompactIntVecBuilder;
|
||||||
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
|
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
|
||||||
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
|
pub use eliasfano::{EliasFano, EliasFanoBuilder};
|
||||||
|
pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
|
||||||
|
pub use intmatrix::{
|
||||||
|
PersistentCompactIntMatrixBuilder, PersistentIntMatrix, pack_compact_int_matrix,
|
||||||
|
};
|
||||||
|
|
||||||
pub use layer_meta::LayerMeta;
|
pub use layer_meta::LayerMeta;
|
||||||
pub use matrix_builder::{ColBuilder, MatrixBuilder};
|
pub use matrix_builder::{ColBuilder, MatrixBuilder};
|
||||||
pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter};
|
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
|
||||||
pub use sparse_intmatrix::{PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder, pack_sparse_compact_int_matrix};
|
pub use reader::{Iter as CompactIntVecIter, PersistentCompactIntVec};
|
||||||
|
pub use sparse_intmatrix::{
|
||||||
|
PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
|
||||||
|
pack_sparse_compact_int_matrix,
|
||||||
|
};
|
||||||
pub use storage_kind::StorageKind;
|
pub use storage_kind::StorageKind;
|
||||||
pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
|
pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
|
||||||
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
|
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
|
||||||
pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials};
|
pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials};
|
||||||
pub use views::{BitSliceView, BitSliceIter, IntSliceView, IntSliceViewIter};
|
pub use views::{BitSliceIter, BitSliceView, IntSliceView, IntSliceViewIter};
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[path = "tests/mod.rs"]
|
#[path = "tests/mod.rs"]
|
||||||
|
|||||||
@@ -35,11 +35,17 @@ use crate::builder::PersistentCompactIntVecBuilder;
|
|||||||
use crate::eliasfano::{EliasFano, EliasFanoBuilder};
|
use crate::eliasfano::{EliasFano, EliasFanoBuilder};
|
||||||
use crate::reader::PersistentCompactIntVec;
|
use crate::reader::PersistentCompactIntVec;
|
||||||
use crate::traits::{BitPartials, ColumnWeights, CountPartials};
|
use crate::traits::{BitPartials, ColumnWeights, CountPartials};
|
||||||
use crate::{PersistentCompactIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
|
use crate::{PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
|
||||||
|
|
||||||
fn singleton_values_path(dir: &Path) -> PathBuf { dir.join("singleton_values.pciv") }
|
fn singleton_values_path(dir: &Path) -> PathBuf {
|
||||||
fn multi_values_path(dir: &Path) -> PathBuf { dir.join("multi_values.pciv") }
|
dir.join("singleton_values.pciv")
|
||||||
fn multi_offsets_base(dir: &Path) -> PathBuf { dir.join("multi_offsets") }
|
}
|
||||||
|
fn multi_values_path(dir: &Path) -> PathBuf {
|
||||||
|
dir.join("multi_values.pciv")
|
||||||
|
}
|
||||||
|
fn multi_offsets_base(dir: &Path) -> PathBuf {
|
||||||
|
dir.join("multi_offsets")
|
||||||
|
}
|
||||||
|
|
||||||
/// Lightweight disk probe: true iff a sparse count matrix has already been
|
/// Lightweight disk probe: true iff a sparse count matrix has already been
|
||||||
/// written at `dir` — the marker [`PersistentCompactIntMatrix::open`]/
|
/// written at `dir` — the marker [`PersistentCompactIntMatrix::open`]/
|
||||||
@@ -73,9 +79,13 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n(&self) -> usize { self.support.n() }
|
pub fn n(&self) -> usize {
|
||||||
|
self.support.n()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn n_cols(&self) -> usize { self.support.n_cols() }
|
pub fn n_cols(&self) -> usize {
|
||||||
|
self.support.n_cols()
|
||||||
|
}
|
||||||
|
|
||||||
/// Calls `f(col, value)` once per non-zero column at `slot`, in
|
/// Calls `f(col, value)` once per non-zero column at `slot`, in
|
||||||
/// ascending column order — the shared decode branch behind every
|
/// ascending column order — the shared decode branch behind every
|
||||||
@@ -104,7 +114,11 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
/// absent).
|
/// absent).
|
||||||
pub fn get(&self, c: usize, slot: usize) -> u32 {
|
pub fn get(&self, c: usize, slot: usize) -> u32 {
|
||||||
let mut found = 0u32;
|
let mut found = 0u32;
|
||||||
self.for_each_cell_in_row(slot, |g, v| if g == c { found = v; });
|
self.for_each_cell_in_row(slot, |g, v| {
|
||||||
|
if g == c {
|
||||||
|
found = v;
|
||||||
|
}
|
||||||
|
});
|
||||||
found
|
found
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,13 +152,17 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
/// Yields every `(idx into slots, col, value)` triple, row by row, in
|
/// Yields every `(idx into slots, col, value)` triple, row by row, in
|
||||||
/// `slots` order — mirrors
|
/// `slots` order — mirrors
|
||||||
/// [`PersistentSparseBitMatrix::nonzero_iter`].
|
/// [`PersistentSparseBitMatrix::nonzero_iter`].
|
||||||
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator<Item = (usize, usize, u32)> + 'a {
|
pub fn nonzero_iter<'a>(
|
||||||
|
&'a self,
|
||||||
|
slots: &'a [usize],
|
||||||
|
) -> impl Iterator<Item = (usize, usize, u32)> + 'a {
|
||||||
let mut next_slot = 0usize;
|
let mut next_slot = 0usize;
|
||||||
let mut cur_idx = 0usize;
|
let mut cur_idx = 0usize;
|
||||||
let mut buf: Vec<(usize, u32)> = Vec::new();
|
let mut buf: Vec<(usize, u32)> = Vec::new();
|
||||||
let mut buf_pos = 0usize;
|
let mut buf_pos = 0usize;
|
||||||
|
|
||||||
std::iter::from_fn(move || loop {
|
std::iter::from_fn(move || {
|
||||||
|
loop {
|
||||||
if buf_pos < buf.len() {
|
if buf_pos < buf.len() {
|
||||||
let (g, v) = buf[buf_pos];
|
let (g, v) = buf[buf_pos];
|
||||||
buf_pos += 1;
|
buf_pos += 1;
|
||||||
@@ -159,6 +177,7 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
buf.clear();
|
buf.clear();
|
||||||
self.for_each_cell_in_row(slot, |g, v| buf.push((g, v)));
|
self.for_each_cell_in_row(slot, |g, v| buf.push((g, v)));
|
||||||
buf_pos = 0;
|
buf_pos = 0;
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -203,7 +222,11 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
||||||
let mut counts = vec![0u64; self.n_cols()];
|
let mut counts = vec![0u64; self.n_cols()];
|
||||||
for slot in 0..self.n() {
|
for slot in 0..self.n() {
|
||||||
self.for_each_cell_in_row(slot, |g, v| if v >= threshold { counts[g] += 1; });
|
self.for_each_cell_in_row(slot, |g, v| {
|
||||||
|
if v >= threshold {
|
||||||
|
counts[g] += 1;
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
Array1::from(counts)
|
Array1::from(counts)
|
||||||
}
|
}
|
||||||
@@ -261,9 +284,13 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
|
|
||||||
impl ColumnWeights for PersistentSparseCompactIntMatrix {
|
impl ColumnWeights for PersistentSparseCompactIntMatrix {
|
||||||
#[inline]
|
#[inline]
|
||||||
fn col_weights(&self) -> Array1<u64> { self.sum() }
|
fn col_weights(&self) -> Array1<u64> {
|
||||||
|
self.sum()
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() }
|
fn partial_kmer_counts(&self) -> Array1<u64> {
|
||||||
|
self.count_nonzero()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CountPartials for PersistentSparseCompactIntMatrix {
|
impl CountPartials for PersistentSparseCompactIntMatrix {
|
||||||
@@ -278,7 +305,9 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
|
|||||||
fn partial_bray(&self) -> Array2<u64> {
|
fn partial_bray(&self) -> Array2<u64> {
|
||||||
let mut m = self.row_major_pairwise(|_, a, _, b| (a as u64).min(b as u64));
|
let mut m = self.row_major_pairwise(|_, a, _, b| (a as u64).min(b as u64));
|
||||||
let sum = self.sum();
|
let sum = self.sum();
|
||||||
for i in 0..self.n_cols() { m[[i, i]] = sum[i]; }
|
for i in 0..self.n_cols() {
|
||||||
|
m[[i, i]] = sum[i];
|
||||||
|
}
|
||||||
m
|
m
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -329,7 +358,8 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
|
|||||||
// left to patch.
|
// left to patch.
|
||||||
return BitPartials::partial_jaccard(&self.support);
|
return BitPartials::partial_jaccard(&self.support);
|
||||||
}
|
}
|
||||||
let mut inter = self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64);
|
let mut inter =
|
||||||
|
self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64);
|
||||||
let mut union = Array2::<u64>::zeros((n, n));
|
let mut union = Array2::<u64>::zeros((n, n));
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
@@ -363,7 +393,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
|
|||||||
});
|
});
|
||||||
let sum = self.sum();
|
let sum = self.sum();
|
||||||
for i in 0..self.n_cols() {
|
for i in 0..self.n_cols() {
|
||||||
m[[i, i]] = if global[i] > 0 { sum[i] as f64 / global[i] as f64 } else { 0.0 };
|
m[[i, i]] = if global[i] > 0 {
|
||||||
|
sum[i] as f64 / global[i] as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
}
|
}
|
||||||
m
|
m
|
||||||
}
|
}
|
||||||
@@ -382,9 +416,21 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
|
|||||||
if i != j {
|
if i != j {
|
||||||
let sa = global[i] as f64;
|
let sa = global[i] as f64;
|
||||||
let sb = global[j] as f64;
|
let sb = global[j] as f64;
|
||||||
let sq_a = if sa > 0.0 { sq[i] as f64 / (sa * sa) } else { 0.0 };
|
let sq_a = if sa > 0.0 {
|
||||||
let sq_b = if sb > 0.0 { sq[j] as f64 / (sb * sb) } else { 0.0 };
|
sq[i] as f64 / (sa * sa)
|
||||||
let cross = if sa > 0.0 && sb > 0.0 { dot[[i, j]] / (sa * sb) } else { 0.0 };
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
let sq_b = if sb > 0.0 {
|
||||||
|
sq[j] as f64 / (sb * sb)
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
let cross = if sa > 0.0 && sb > 0.0 {
|
||||||
|
dot[[i, j]] / (sa * sb)
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
m[[i, j]] = sq_a + sq_b - 2.0 * cross;
|
m[[i, j]] = sq_a + sq_b - 2.0 * cross;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -409,7 +455,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix {
|
|||||||
let sb = global[j] as f64;
|
let sb = global[j] as f64;
|
||||||
let pa_sum = if sa > 0.0 { sum[i] as f64 / sa } else { 0.0 };
|
let pa_sum = if sa > 0.0 { sum[i] as f64 / sa } else { 0.0 };
|
||||||
let pb_sum = if sb > 0.0 { sum[j] as f64 / sb } else { 0.0 };
|
let pb_sum = if sb > 0.0 { sum[j] as f64 / sb } else { 0.0 };
|
||||||
let cross = if sa > 0.0 && sb > 0.0 { sqrt_dot[[i, j]] / (sa * sb).sqrt() } else { 0.0 };
|
let cross = if sa > 0.0 && sb > 0.0 {
|
||||||
|
sqrt_dot[[i, j]] / (sa * sb).sqrt()
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
m[[i, j]] = pa_sum + pb_sum - 2.0 * cross;
|
m[[i, j]] = pa_sum + pb_sum - 2.0 * cross;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -461,7 +511,6 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
|||||||
self.support.push_row(cols);
|
self.support.push_row(cols);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// Builds a sparse matrix from an already-built dense
|
/// Builds a sparse matrix from an already-built dense
|
||||||
/// [`PersistentCompactIntMatrix`] (`Columnar` or `Packed`) — same
|
/// [`PersistentCompactIntMatrix`] (`Columnar` or `Packed`) — same
|
||||||
/// batched-transpose shape as
|
/// batched-transpose shape as
|
||||||
@@ -475,7 +524,7 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
|||||||
/// pairs are appended to its own buffer in strictly ascending column
|
/// pairs are appended to its own buffer in strictly ascending column
|
||||||
/// order "for free", matching `push_row`'s ascending-`cols` contract
|
/// order "for free", matching `push_row`'s ascending-`cols` contract
|
||||||
/// without an explicit sort.
|
/// without an explicit sort.
|
||||||
pub fn build_from_dense(dense: &PersistentCompactIntMatrix, dir: &Path) -> io::Result<Self> {
|
pub fn build_from_dense(dense: &PersistentIntMatrix, dir: &Path) -> io::Result<Self> {
|
||||||
let n = dense.n();
|
let n = dense.n();
|
||||||
let n_cols = dense.n_cols();
|
let n_cols = dense.n_cols();
|
||||||
let mut builder = Self::new(n, n_cols, dir)?;
|
let mut builder = Self::new(n, n_cols, dir)?;
|
||||||
@@ -500,23 +549,34 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn close(self) -> io::Result<()> {
|
pub fn close(self) -> io::Result<()> {
|
||||||
let Self { support, dir, singleton_values, multi_values, multi_row_offsets } = self;
|
let Self {
|
||||||
|
support,
|
||||||
|
dir,
|
||||||
|
singleton_values,
|
||||||
|
multi_values,
|
||||||
|
multi_row_offsets,
|
||||||
|
} = self;
|
||||||
support.close()?;
|
support.close()?;
|
||||||
|
|
||||||
let mut sv = PersistentCompactIntVecBuilder::new(singleton_values.len(), &singleton_values_path(&dir))?;
|
let mut sv = PersistentCompactIntVecBuilder::new(
|
||||||
|
singleton_values.len(),
|
||||||
|
&singleton_values_path(&dir),
|
||||||
|
)?;
|
||||||
for (i, &v) in singleton_values.iter().enumerate() {
|
for (i, &v) in singleton_values.iter().enumerate() {
|
||||||
sv.set(i, v);
|
sv.set(i, v);
|
||||||
}
|
}
|
||||||
sv.close()?;
|
sv.close()?;
|
||||||
|
|
||||||
let mut mv = PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?;
|
let mut mv =
|
||||||
|
PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?;
|
||||||
for (i, &v) in multi_values.iter().enumerate() {
|
for (i, &v) in multi_values.iter().enumerate() {
|
||||||
mv.set(i, v);
|
mv.set(i, v);
|
||||||
}
|
}
|
||||||
mv.close()?;
|
mv.close()?;
|
||||||
|
|
||||||
let universe = multi_values.len() as u64 + 1;
|
let universe = multi_values.len() as u64 + 1;
|
||||||
let mut mo = EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?;
|
let mut mo =
|
||||||
|
EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?;
|
||||||
for &o in &multi_row_offsets {
|
for &o in &multi_row_offsets {
|
||||||
mo.push(o);
|
mo.push(o);
|
||||||
}
|
}
|
||||||
@@ -544,7 +604,7 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
|||||||
/// half-written one). Idempotent — a no-op if `singleton_values.pciv`
|
/// half-written one). Idempotent — a no-op if `singleton_values.pciv`
|
||||||
/// already exists.
|
/// already exists.
|
||||||
pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
||||||
use crate::intmatrix::{col_path, ColumnarCompactIntMatrix, PackedCompactIntMatrix};
|
use crate::intmatrix::{ColumnarCompactIntMatrix, PackedCompactIntMatrix, col_path};
|
||||||
|
|
||||||
if is_present(dir) {
|
if is_present(dir) {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -552,12 +612,12 @@ pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
|||||||
|
|
||||||
let packed_path = dir.join("matrix.pcmx");
|
let packed_path = dir.join("matrix.pcmx");
|
||||||
if packed_path.exists() {
|
if packed_path.exists() {
|
||||||
let dense = PersistentCompactIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?);
|
let dense = PersistentIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?);
|
||||||
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
||||||
drop(dense);
|
drop(dense);
|
||||||
fs::remove_file(&packed_path)?;
|
fs::remove_file(&packed_path)?;
|
||||||
} else {
|
} else {
|
||||||
let dense = PersistentCompactIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?);
|
let dense = PersistentIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?);
|
||||||
let n_cols = dense.n_cols();
|
let n_cols = dense.n_cols();
|
||||||
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
|
||||||
drop(dense);
|
drop(dense);
|
||||||
|
|||||||
@@ -1,25 +1,26 @@
|
|||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
ColGroup, MatrixGroupOps,
|
ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder,
|
||||||
PersistentBitMatrix, PersistentBitMatrixBuilder,
|
PersistentIntMatrix, PersistentCompactIntMatrixBuilder,
|
||||||
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
|
|
||||||
};
|
};
|
||||||
use crate::{PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder};
|
use crate::{PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder};
|
||||||
|
|
||||||
// ── helpers ───────────────────────────────────────────────────────────────────
|
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
|
fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
|
||||||
let n = cols.first().map_or(0, |c| c.len());
|
let n = cols.first().map_or(0, |c| c.len());
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
|
||||||
for &col in cols {
|
for &col in cols {
|
||||||
let mut cb = b.add_col().unwrap();
|
let mut cb = b.add_col().unwrap();
|
||||||
for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
|
for (slot, &v) in col.iter().enumerate() {
|
||||||
|
cb.set(slot, v);
|
||||||
|
}
|
||||||
cb.close().unwrap();
|
cb.close().unwrap();
|
||||||
}
|
}
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
(dir, m)
|
(dir, m)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -30,7 +31,9 @@ fn make_bit_matrix(cols: &[&[bool]]) -> (tempfile::TempDir, PersistentBitMatrix)
|
|||||||
let mut b = PersistentBitMatrixBuilder::new(n, &presence).unwrap();
|
let mut b = PersistentBitMatrixBuilder::new(n, &presence).unwrap();
|
||||||
for &col in cols {
|
for &col in cols {
|
||||||
let mut cb = b.add_col().unwrap();
|
let mut cb = b.add_col().unwrap();
|
||||||
for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); }
|
for (slot, &v) in col.iter().enumerate() {
|
||||||
|
cb.set(slot, v);
|
||||||
|
}
|
||||||
cb.close().unwrap();
|
cb.close().unwrap();
|
||||||
}
|
}
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
@@ -115,9 +118,13 @@ fn mask_with_zeros_selected_slots() {
|
|||||||
// count vec [10, 20, 30, 40], mask [T, F, T, F] → [10, 0, 30, 0]
|
// count vec [10, 20, 30, 40], mask [T, F, T, F] → [10, 0, 30, 0]
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
|
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
|
||||||
v.set(0, 10); v.set(1, 20); v.set(2, 30); v.set(3, 40);
|
v.set(0, 10);
|
||||||
|
v.set(1, 20);
|
||||||
|
v.set(2, 30);
|
||||||
|
v.set(3, 40);
|
||||||
let mut mask = PersistentBitVecBuilder::new(4, &dir.path().join("m.pbiv")).unwrap();
|
let mut mask = PersistentBitVecBuilder::new(4, &dir.path().join("m.pbiv")).unwrap();
|
||||||
mask.set(0, true); mask.set(2, true);
|
mask.set(0, true);
|
||||||
|
mask.set(2, true);
|
||||||
v.mask_with(mask.view());
|
v.mask_with(mask.view());
|
||||||
v.close().unwrap();
|
v.close().unwrap();
|
||||||
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
|
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
|
||||||
@@ -132,9 +139,12 @@ fn mask_with_overflow_slot_zeroed() {
|
|||||||
// overflow slot (value 500) masked out → removed from overflow, primary=0
|
// overflow slot (value 500) masked out → removed from overflow, primary=0
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut v = PersistentCompactIntVecBuilder::new(3, &dir.path().join("v.pciv")).unwrap();
|
let mut v = PersistentCompactIntVecBuilder::new(3, &dir.path().join("v.pciv")).unwrap();
|
||||||
v.set(0, 10); v.set(1, 500); v.set(2, 5);
|
v.set(0, 10);
|
||||||
|
v.set(1, 500);
|
||||||
|
v.set(2, 5);
|
||||||
let mut mask = PersistentBitVecBuilder::new(3, &dir.path().join("m.pbiv")).unwrap();
|
let mut mask = PersistentBitVecBuilder::new(3, &dir.path().join("m.pbiv")).unwrap();
|
||||||
mask.set(0, true); mask.set(2, true); // slot 1 masked out
|
mask.set(0, true);
|
||||||
|
mask.set(2, true); // slot 1 masked out
|
||||||
v.mask_with(mask.view());
|
v.mask_with(mask.view());
|
||||||
v.close().unwrap();
|
v.close().unwrap();
|
||||||
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
|
let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap();
|
||||||
@@ -142,14 +152,20 @@ fn mask_with_overflow_slot_zeroed() {
|
|||||||
assert_eq!(r.get(1), 0);
|
assert_eq!(r.get(1), 0);
|
||||||
assert_eq!(r.get(2), 5);
|
assert_eq!(r.get(2), 5);
|
||||||
let ov: Vec<_> = r.view().overflow_entries().collect();
|
let ov: Vec<_> = r.view().overflow_entries().collect();
|
||||||
assert!(ov.is_empty(), "overflow entry for masked-out slot should be gone");
|
assert!(
|
||||||
|
ov.is_empty(),
|
||||||
|
"overflow entry for masked-out slot should be gone"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mask_with_all_ones_is_noop() {
|
fn mask_with_all_ones_is_noop() {
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
|
let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap();
|
||||||
v.set(0, 300); v.set(1, 1); v.set(2, 0); v.set(3, 42);
|
v.set(0, 300);
|
||||||
|
v.set(1, 1);
|
||||||
|
v.set(2, 0);
|
||||||
|
v.set(3, 42);
|
||||||
let mask = PersistentBitVecBuilder::new_ones(4, &dir.path().join("m.pbiv")).unwrap();
|
let mask = PersistentBitVecBuilder::new_ones(4, &dir.path().join("m.pbiv")).unwrap();
|
||||||
v.mask_with(mask.view());
|
v.mask_with(mask.view());
|
||||||
v.close().unwrap();
|
v.close().unwrap();
|
||||||
@@ -184,10 +200,7 @@ fn bit_partial_group_presence_count() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn bit_partial_group_any() {
|
fn bit_partial_group_any() {
|
||||||
// col0=[T,F,F], col1=[F,F,T], group {0,1}: any = [T, F, T]
|
// col0=[T,F,F], col1=[F,F,T], group {0,1}: any = [T, F, T]
|
||||||
let (_d, m) = make_bit_matrix(&[
|
let (_d, m) = make_bit_matrix(&[&[true, false, false], &[false, false, true]]);
|
||||||
&[true, false, false],
|
|
||||||
&[false, false, true],
|
|
||||||
]);
|
|
||||||
let g = ColGroup::new("g", vec![0, 1]);
|
let g = ColGroup::new("g", vec![0, 1]);
|
||||||
let result = m.partial_group_any(&g, 1).unwrap();
|
let result = m.partial_group_any(&g, 1).unwrap();
|
||||||
assert_eq!(result.get(0), true);
|
assert_eq!(result.get(0), true);
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
use crate::{pack_compact_int_matrix, pack_sparse_compact_int_matrix, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder, StorageKind};
|
|
||||||
use crate::traits::CountPartials;
|
use crate::traits::CountPartials;
|
||||||
|
use crate::{
|
||||||
|
PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder,
|
||||||
|
PersistentIntMatrix, StorageKind, pack_compact_int_matrix, pack_sparse_compact_int_matrix,
|
||||||
|
};
|
||||||
|
|
||||||
fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
|
fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
|
||||||
let n = cols.first().map_or(0, |c| c.len());
|
let n = cols.first().map_or(0, |c| c.len());
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap();
|
||||||
@@ -15,7 +18,7 @@ fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatri
|
|||||||
cb.close().unwrap();
|
cb.close().unwrap();
|
||||||
}
|
}
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
(dir, m)
|
(dir, m)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -31,7 +34,7 @@ fn single_col_roundtrip() {
|
|||||||
col.close().unwrap();
|
col.close().unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 1);
|
assert_eq!(m.n_cols(), 1);
|
||||||
assert_eq!(m.n(), 4);
|
assert_eq!(m.n(), 4);
|
||||||
assert_eq!(&*m.row(0), &[10u32]);
|
assert_eq!(&*m.row(0), &[10u32]);
|
||||||
@@ -45,14 +48,18 @@ fn two_cols_roundtrip() {
|
|||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::new(3, &dir.path().join("counts")).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::new(3, &dir.path().join("counts")).unwrap();
|
||||||
let mut col0 = b.add_col().unwrap();
|
let mut col0 = b.add_col().unwrap();
|
||||||
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3);
|
col0.set(0, 1);
|
||||||
|
col0.set(1, 2);
|
||||||
|
col0.set(2, 3);
|
||||||
col0.close().unwrap();
|
col0.close().unwrap();
|
||||||
let mut col1 = b.add_col().unwrap();
|
let mut col1 = b.add_col().unwrap();
|
||||||
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30);
|
col1.set(0, 10);
|
||||||
|
col1.set(1, 20);
|
||||||
|
col1.set(2, 30);
|
||||||
col1.close().unwrap();
|
col1.close().unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 2);
|
assert_eq!(m.n_cols(), 2);
|
||||||
assert_eq!(&*m.row(0), &[1u32, 10]);
|
assert_eq!(&*m.row(0), &[1u32, 10]);
|
||||||
assert_eq!(&*m.row(1), &[2u32, 20]);
|
assert_eq!(&*m.row(1), &[2u32, 20]);
|
||||||
@@ -66,7 +73,9 @@ fn resume_continues_n_cols_and_appends_columns() {
|
|||||||
|
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::new(3, &counts).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::new(3, &counts).unwrap();
|
||||||
let mut col0 = b.add_col().unwrap();
|
let mut col0 = b.add_col().unwrap();
|
||||||
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3);
|
col0.set(0, 1);
|
||||||
|
col0.set(1, 2);
|
||||||
|
col0.set(2, 3);
|
||||||
col0.close().unwrap();
|
col0.close().unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
|
|
||||||
@@ -74,11 +83,13 @@ fn resume_continues_n_cols_and_appends_columns() {
|
|||||||
assert_eq!(resumed.n(), 3);
|
assert_eq!(resumed.n(), 3);
|
||||||
assert_eq!(resumed.n_cols(), 1);
|
assert_eq!(resumed.n_cols(), 1);
|
||||||
let mut col1 = resumed.add_col().unwrap();
|
let mut col1 = resumed.add_col().unwrap();
|
||||||
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30);
|
col1.set(0, 10);
|
||||||
|
col1.set(1, 20);
|
||||||
|
col1.set(2, 30);
|
||||||
col1.close().unwrap();
|
col1.close().unwrap();
|
||||||
resumed.close().unwrap();
|
resumed.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 2);
|
assert_eq!(m.n_cols(), 2);
|
||||||
assert_eq!(&*m.row(0), &[1u32, 10]);
|
assert_eq!(&*m.row(0), &[1u32, 10]);
|
||||||
assert_eq!(&*m.row(1), &[2u32, 20]);
|
assert_eq!(&*m.row(1), &[2u32, 20]);
|
||||||
@@ -90,17 +101,21 @@ fn resume_twice_keeps_appending() {
|
|||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let counts = dir.path().join("counts");
|
let counts = dir.path().join("counts");
|
||||||
|
|
||||||
PersistentCompactIntMatrixBuilder::new(2, &counts).unwrap().close().unwrap();
|
PersistentCompactIntMatrixBuilder::new(2, &counts)
|
||||||
|
.unwrap()
|
||||||
|
.close()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
for v in [1u32, 2, 3] {
|
for v in [1u32, 2, 3] {
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap();
|
||||||
let mut col = b.add_col().unwrap();
|
let mut col = b.add_col().unwrap();
|
||||||
col.set(0, v); col.set(1, v * 10);
|
col.set(0, v);
|
||||||
|
col.set(1, v * 10);
|
||||||
col.close().unwrap();
|
col.close().unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 3);
|
assert_eq!(m.n_cols(), 3);
|
||||||
assert_eq!(&*m.row(0), &[1u32, 2, 3]);
|
assert_eq!(&*m.row(0), &[1u32, 2, 3]);
|
||||||
assert_eq!(&*m.row(1), &[10u32, 20, 30]);
|
assert_eq!(&*m.row(1), &[10u32, 20, 30]);
|
||||||
@@ -111,11 +126,12 @@ fn col_accessor() {
|
|||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let mut b = PersistentCompactIntMatrixBuilder::new(2, &dir.path().join("counts")).unwrap();
|
let mut b = PersistentCompactIntMatrixBuilder::new(2, &dir.path().join("counts")).unwrap();
|
||||||
let mut col0 = b.add_col().unwrap();
|
let mut col0 = b.add_col().unwrap();
|
||||||
col0.set(0, 5); col0.set(1, 7);
|
col0.set(0, 5);
|
||||||
|
col0.set(1, 7);
|
||||||
col0.close().unwrap();
|
col0.close().unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.col(0).get(0), 5);
|
assert_eq!(m.col(0).get(0), 5);
|
||||||
assert_eq!(m.col(0).get(1), 7);
|
assert_eq!(m.col(0).get(1), 7);
|
||||||
}
|
}
|
||||||
@@ -126,7 +142,7 @@ fn zero_cols_roundtrip() {
|
|||||||
let b = PersistentCompactIntMatrixBuilder::new(10, &dir.path().join("counts")).unwrap();
|
let b = PersistentCompactIntMatrixBuilder::new(10, &dir.path().join("counts")).unwrap();
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 0);
|
assert_eq!(m.n_cols(), 0);
|
||||||
assert_eq!(m.n(), 10);
|
assert_eq!(m.n(), 10);
|
||||||
}
|
}
|
||||||
@@ -139,7 +155,9 @@ fn bray_dist_matrix_symmetry_and_diagonal() {
|
|||||||
let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]);
|
let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]);
|
||||||
let dm = m.bray_dist_matrix();
|
let dm = m.bray_dist_matrix();
|
||||||
let n = m.n_cols();
|
let n = m.n_cols();
|
||||||
for i in 0..n { assert_eq!(dm[[i, i]], 0.0, "diagonal"); }
|
for i in 0..n {
|
||||||
|
assert_eq!(dm[[i, i]], 0.0, "diagonal");
|
||||||
|
}
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
assert!((dm[[i, j]] - dm[[j, i]]).abs() < 1e-12, "symmetry");
|
assert!((dm[[i, j]] - dm[[j, i]]).abs() < 1e-12, "symmetry");
|
||||||
@@ -189,7 +207,11 @@ fn partial_bray_dist_matrix_consistent() {
|
|||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in i + 1..n {
|
for j in i + 1..n {
|
||||||
let denom = col_sums[i] + col_sums[j];
|
let denom = col_sums[i] + col_sums[j];
|
||||||
let dist = if denom == 0 { 0.0 } else { 1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64 };
|
let dist = if denom == 0 {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64
|
||||||
|
};
|
||||||
let expected = m.col(i).bray_dist(m.col(j));
|
let expected = m.col(i).bray_dist(m.col(j));
|
||||||
assert!((dist - expected).abs() < 1e-12, "[{i},{j}]");
|
assert!((dist - expected).abs() < 1e-12, "[{i},{j}]");
|
||||||
}
|
}
|
||||||
@@ -254,8 +276,13 @@ fn partial_relfreq_bray_matches_full() {
|
|||||||
// partial[i,j] = sum_min_relfreq; full[i,j] = 1 - sum_min_relfreq (off-diagonal only)
|
// partial[i,j] = sum_min_relfreq; full[i,j] = 1 - sum_min_relfreq (off-diagonal only)
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
if i == j { continue; }
|
if i == j {
|
||||||
assert!((partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12, "[{i},{j}]");
|
continue;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
(partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12,
|
||||||
|
"[{i},{j}]"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -270,7 +297,10 @@ fn partial_relfreq_euclidean_matches_full() {
|
|||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
// partial = squared euclidean; full = sqrt(partial)
|
// partial = squared euclidean; full = sqrt(partial)
|
||||||
assert!((partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12, "[{i},{j}]");
|
assert!(
|
||||||
|
(partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12,
|
||||||
|
"[{i},{j}]"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -285,7 +315,10 @@ fn partial_hellinger_matches_full() {
|
|||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
// partial / sqrt(2) gives the Hellinger distance
|
// partial / sqrt(2) gives the Hellinger distance
|
||||||
assert!((partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12, "[{i},{j}]");
|
assert!(
|
||||||
|
(partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12,
|
||||||
|
"[{i},{j}]"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -295,7 +328,7 @@ fn col_view_packed_values() {
|
|||||||
// Build Columnar with overflow values (≥ 255), pack, reopen as Packed, exercise col_view().
|
// Build Columnar with overflow values (≥ 255), pack, reopen as Packed, exercise col_view().
|
||||||
let (dir, _col) = make_matrix(&[&[10, 300, 500], &[200, 50, 1000]]);
|
let (dir, _col) = make_matrix(&[&[10, 300, 500], &[200, 50, 1000]]);
|
||||||
pack_compact_int_matrix(&dir.path().join("counts")).unwrap();
|
pack_compact_int_matrix(&dir.path().join("counts")).unwrap();
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
|
|
||||||
// col 0: [10, 300, 500] — two overflow slots
|
// col 0: [10, 300, 500] — two overflow slots
|
||||||
let v0 = m.col_view(0);
|
let v0 = m.col_view(0);
|
||||||
@@ -326,7 +359,7 @@ fn col_view_packed_matches_columnar() {
|
|||||||
// Re-build in a separate dir so we can pack without touching m_col's files.
|
// Re-build in a separate dir so we can pack without touching m_col's files.
|
||||||
let (dir_pack, _) = make_matrix(data);
|
let (dir_pack, _) = make_matrix(data);
|
||||||
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
|
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
|
||||||
let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap();
|
let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap();
|
||||||
|
|
||||||
for c in 0..data.len() {
|
for c in 0..data.len() {
|
||||||
let col_ref = m_col.col(c);
|
let col_ref = m_col.col(c);
|
||||||
@@ -355,7 +388,7 @@ fn nonzero_iter_matches_row() {
|
|||||||
let (dir_col, m_col) = make_matrix(data);
|
let (dir_col, m_col) = make_matrix(data);
|
||||||
let (dir_pack, _) = make_matrix(data);
|
let (dir_pack, _) = make_matrix(data);
|
||||||
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
|
pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap();
|
||||||
let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap();
|
let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap();
|
||||||
|
|
||||||
let slots = [4usize, 0, 3, 1];
|
let slots = [4usize, 0, 3, 1];
|
||||||
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
|
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
|
||||||
@@ -390,7 +423,7 @@ fn sparse_roundtrip_matches_columnar() {
|
|||||||
let (dir_col, m_col) = make_matrix(data);
|
let (dir_col, m_col) = make_matrix(data);
|
||||||
let (dir_sparse, _) = make_matrix(data);
|
let (dir_sparse, _) = make_matrix(data);
|
||||||
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
||||||
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
|
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
|
||||||
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
||||||
|
|
||||||
assert_eq!(m_sparse.n(), m_col.n());
|
assert_eq!(m_sparse.n(), m_col.n());
|
||||||
@@ -441,7 +474,7 @@ fn sparse_count_partials_match_dense() {
|
|||||||
let (dir_col, m_col) = make_matrix(data);
|
let (dir_col, m_col) = make_matrix(data);
|
||||||
let (dir_sparse, _) = make_matrix(data);
|
let (dir_sparse, _) = make_matrix(data);
|
||||||
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
||||||
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
|
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
|
||||||
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
||||||
|
|
||||||
let n = m_col.n_cols();
|
let n = m_col.n_cols();
|
||||||
@@ -461,7 +494,11 @@ fn sparse_count_partials_match_dense() {
|
|||||||
let eucl_sparse = m_sparse.partial_euclidean_dist_matrix();
|
let eucl_sparse = m_sparse.partial_euclidean_dist_matrix();
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
close(eucl_col[[i, j]], eucl_sparse[[i, j]], &format!("partial_euclidean[{i},{j}]"));
|
close(
|
||||||
|
eucl_col[[i, j]],
|
||||||
|
eucl_sparse[[i, j]],
|
||||||
|
&format!("partial_euclidean[{i},{j}]"),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -469,9 +506,16 @@ fn sparse_count_partials_match_dense() {
|
|||||||
// (support shortcut), and >1 (general row-major path).
|
// (support shortcut), and >1 (general row-major path).
|
||||||
for threshold in [0u32, 1, 2, 3] {
|
for threshold in [0u32, 1, 2, 3] {
|
||||||
let (inter_col, union_col) = m_col.partial_threshold_jaccard_dist_matrix(threshold);
|
let (inter_col, union_col) = m_col.partial_threshold_jaccard_dist_matrix(threshold);
|
||||||
let (inter_sparse, union_sparse) = m_sparse.partial_threshold_jaccard_dist_matrix(threshold);
|
let (inter_sparse, union_sparse) =
|
||||||
assert_eq!(inter_col, inter_sparse, "partial_threshold_jaccard inter, threshold={threshold}");
|
m_sparse.partial_threshold_jaccard_dist_matrix(threshold);
|
||||||
assert_eq!(union_col, union_sparse, "partial_threshold_jaccard union, threshold={threshold}");
|
assert_eq!(
|
||||||
|
inter_col, inter_sparse,
|
||||||
|
"partial_threshold_jaccard inter, threshold={threshold}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
union_col, union_sparse,
|
||||||
|
"partial_threshold_jaccard union, threshold={threshold}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// partial_relfreq_bray
|
// partial_relfreq_bray
|
||||||
@@ -479,7 +523,11 @@ fn sparse_count_partials_match_dense() {
|
|||||||
let rfb_sparse = m_sparse.partial_relfreq_bray_dist_matrix(&global);
|
let rfb_sparse = m_sparse.partial_relfreq_bray_dist_matrix(&global);
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
close(rfb_col[[i, j]], rfb_sparse[[i, j]], &format!("partial_relfreq_bray[{i},{j}]"));
|
close(
|
||||||
|
rfb_col[[i, j]],
|
||||||
|
rfb_sparse[[i, j]],
|
||||||
|
&format!("partial_relfreq_bray[{i},{j}]"),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -488,7 +536,11 @@ fn sparse_count_partials_match_dense() {
|
|||||||
let rfe_sparse = m_sparse.partial_relfreq_euclidean_dist_matrix(&global);
|
let rfe_sparse = m_sparse.partial_relfreq_euclidean_dist_matrix(&global);
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
close(rfe_col[[i, j]], rfe_sparse[[i, j]], &format!("partial_relfreq_euclidean[{i},{j}]"));
|
close(
|
||||||
|
rfe_col[[i, j]],
|
||||||
|
rfe_sparse[[i, j]],
|
||||||
|
&format!("partial_relfreq_euclidean[{i},{j}]"),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -497,7 +549,11 @@ fn sparse_count_partials_match_dense() {
|
|||||||
let hel_sparse = m_sparse.partial_hellinger_euclidean_dist_matrix(&global);
|
let hel_sparse = m_sparse.partial_hellinger_euclidean_dist_matrix(&global);
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
close(hel_col[[i, j]], hel_sparse[[i, j]], &format!("partial_hellinger[{i},{j}]"));
|
close(
|
||||||
|
hel_col[[i, j]],
|
||||||
|
hel_sparse[[i, j]],
|
||||||
|
&format!("partial_hellinger[{i},{j}]"),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -514,7 +570,7 @@ fn sparse_roundtrip_from_packed() {
|
|||||||
let (dir_sparse, _) = make_matrix(data);
|
let (dir_sparse, _) = make_matrix(data);
|
||||||
pack_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
pack_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
||||||
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap();
|
||||||
let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap();
|
let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap();
|
||||||
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse);
|
||||||
for slot in 0..m_col.n() {
|
for slot in 0..m_col.n() {
|
||||||
assert_eq!(&*m_sparse.row(slot), &*m_col.row(slot), "slot={slot}");
|
assert_eq!(&*m_sparse.row(slot), &*m_col.row(slot), "slot={slot}");
|
||||||
@@ -540,7 +596,10 @@ fn partial_relfreq_bray_additive_across_split() {
|
|||||||
let n = m_full.n_cols();
|
let n = m_full.n_cols();
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
assert!((combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12, "[{i},{j}]");
|
assert!(
|
||||||
|
(combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12,
|
||||||
|
"[{i},{j}]"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -551,7 +610,9 @@ fn make_pciv(counts: &[u32]) -> (tempfile::TempDir, PersistentCompactIntVec) {
|
|||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let path = dir.path().join("c.pciv");
|
let path = dir.path().join("c.pciv");
|
||||||
let mut b = PersistentCompactIntVecBuilder::new(counts.len(), &path).unwrap();
|
let mut b = PersistentCompactIntVecBuilder::new(counts.len(), &path).unwrap();
|
||||||
for (i, &v) in counts.iter().enumerate() { b.set(i, v); }
|
for (i, &v) in counts.iter().enumerate() {
|
||||||
|
b.set(i, v);
|
||||||
|
}
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
let r = PersistentCompactIntVec::open(&path).unwrap();
|
let r = PersistentCompactIntVec::open(&path).unwrap();
|
||||||
(dir, r)
|
(dir, r)
|
||||||
@@ -592,7 +653,10 @@ fn pciv_collect_slots_values_empty() {
|
|||||||
fn pciv_collect_slots_values_out_of_bounds_panics() {
|
fn pciv_collect_slots_values_out_of_bounds_panics() {
|
||||||
let (_dir, v) = make_pciv(&[10u32, 20]);
|
let (_dir, v) = make_pciv(&[10u32, 20]);
|
||||||
let result = std::panic::catch_unwind(|| v.collect_slots_values(&[0, 2]));
|
let result = std::panic::catch_unwind(|| v.collect_slots_values(&[0, 2]));
|
||||||
assert!(result.is_err(), "collect_slots_values should panic on out-of-bounds slot");
|
assert!(
|
||||||
|
result.is_err(),
|
||||||
|
"collect_slots_values should panic on out-of-bounds slot"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// IntSliceView collect_slots_values (same logic, exercised through the view)
|
// IntSliceView collect_slots_values (same logic, exercised through the view)
|
||||||
|
|||||||
@@ -1,13 +1,14 @@
|
|||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
pack_compact_int_matrix, pack_sparse_compact_int_matrix, ColumnWeights, PersistentCompactIntMatrix,
|
ColumnWeights, PersistentIntMatrix, PersistentCompactIntMatrixBuilder,
|
||||||
PersistentCompactIntMatrixBuilder, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
|
PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder,
|
||||||
|
pack_compact_int_matrix, pack_sparse_compact_int_matrix,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Builds a dense `PersistentCompactIntMatrix` from column-major `u32` data
|
/// Builds a dense `PersistentCompactIntMatrix` from column-major `u32` data
|
||||||
/// — mirrors `tests/intmatrix.rs`'s own `make_matrix` helper.
|
/// — mirrors `tests/intmatrix.rs`'s own `make_matrix` helper.
|
||||||
fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
|
fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) {
|
||||||
let n = cols.first().map_or(0, |c| c.len());
|
let n = cols.first().map_or(0, |c| c.len());
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let counts_dir = dir.path().join("counts");
|
let counts_dir = dir.path().join("counts");
|
||||||
@@ -20,17 +21,21 @@ fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix
|
|||||||
cb.close().unwrap();
|
cb.close().unwrap();
|
||||||
}
|
}
|
||||||
b.close().unwrap();
|
b.close().unwrap();
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
(dir, m)
|
(dir, m)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Builds a sparse int matrix directly from row-major `u32` data (one
|
/// Builds a sparse int matrix directly from row-major `u32` data (one
|
||||||
/// slice per row, `n_cols` values each) — mirrors `tests/sparse.rs`'s
|
/// slice per row, `n_cols` values each) — mirrors `tests/sparse.rs`'s
|
||||||
/// `make_sparse` helper.
|
/// `make_sparse` helper.
|
||||||
fn make_sparse(rows: &[&[u32]], n_cols: usize) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) {
|
fn make_sparse(
|
||||||
|
rows: &[&[u32]],
|
||||||
|
n_cols: usize,
|
||||||
|
) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) {
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let sparse_dir = dir.path().join("sparse");
|
let sparse_dir = dir.path().join("sparse");
|
||||||
let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap();
|
let mut b =
|
||||||
|
PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap();
|
||||||
let mut cols = Vec::new();
|
let mut cols = Vec::new();
|
||||||
let mut values = Vec::new();
|
let mut values = Vec::new();
|
||||||
for row in rows {
|
for row in rows {
|
||||||
@@ -70,11 +75,7 @@ fn basic_roundtrip_singletons_and_multi() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn get_matches_row() {
|
fn get_matches_row() {
|
||||||
let rows: Vec<&[u32]> = vec![
|
let rows: Vec<&[u32]> = vec![&[3, 0, 5], &[0, 8, 0], &[1, 2, 4]];
|
||||||
&[3, 0, 5],
|
|
||||||
&[0, 8, 0],
|
|
||||||
&[1, 2, 4],
|
|
||||||
];
|
|
||||||
let (_dir, m) = make_sparse(&rows, 3);
|
let (_dir, m) = make_sparse(&rows, 3);
|
||||||
for (slot, row) in rows.iter().enumerate() {
|
for (slot, row) in rows.iter().enumerate() {
|
||||||
for (c, &expected) in row.iter().enumerate() {
|
for (c, &expected) in row.iter().enumerate() {
|
||||||
@@ -85,11 +86,7 @@ fn get_matches_row() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn fill_row_matches_row() {
|
fn fill_row_matches_row() {
|
||||||
let rows: Vec<&[u32]> = vec![
|
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]];
|
||||||
&[9, 0, 2],
|
|
||||||
&[0, 4, 0],
|
|
||||||
&[1, 1, 1],
|
|
||||||
];
|
|
||||||
let (_dir, m) = make_sparse(&rows, 3);
|
let (_dir, m) = make_sparse(&rows, 3);
|
||||||
let mut buf = vec![0u32; 3];
|
let mut buf = vec![0u32; 3];
|
||||||
for slot in 0..3 {
|
for slot in 0..3 {
|
||||||
@@ -120,11 +117,7 @@ fn fill_sub_matrix_matches_row() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn nonzero_iter_matches_row() {
|
fn nonzero_iter_matches_row() {
|
||||||
let rows: Vec<&[u32]> = vec![
|
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]];
|
||||||
&[9, 0, 2],
|
|
||||||
&[0, 4, 0],
|
|
||||||
&[1, 1, 1],
|
|
||||||
];
|
|
||||||
let (_dir, m) = make_sparse(&rows, 3);
|
let (_dir, m) = make_sparse(&rows, 3);
|
||||||
let slots = [2usize, 0, 1];
|
let slots = [2usize, 0, 1];
|
||||||
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
|
let mut expected: Vec<(usize, usize, u32)> = Vec::new();
|
||||||
@@ -143,12 +136,7 @@ fn nonzero_iter_matches_row() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn sum_and_count_nonzero_match_naive() {
|
fn sum_and_count_nonzero_match_naive() {
|
||||||
let rows: Vec<&[u32]> = vec![
|
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1], &[0, 0, 6]];
|
||||||
&[9, 0, 2],
|
|
||||||
&[0, 4, 0],
|
|
||||||
&[1, 1, 1],
|
|
||||||
&[0, 0, 6],
|
|
||||||
];
|
|
||||||
let (_dir, m) = make_sparse(&rows, 3);
|
let (_dir, m) = make_sparse(&rows, 3);
|
||||||
let mut expected_sum = [0u64; 3];
|
let mut expected_sum = [0u64; 3];
|
||||||
let mut expected_count = [0u64; 3];
|
let mut expected_count = [0u64; 3];
|
||||||
@@ -193,14 +181,21 @@ fn reopen_after_close_matches_original() {
|
|||||||
let rows: Vec<Vec<u32>> = (0..500)
|
let rows: Vec<Vec<u32>> = (0..500)
|
||||||
.map(|i| {
|
.map(|i| {
|
||||||
(0..37)
|
(0..37)
|
||||||
.map(|c| if (i * 7 + c * 3) % 11 == 0 { ((i + c) % 250 + 1) as u32 } else { 0 })
|
.map(|c| {
|
||||||
|
if (i * 7 + c * 3) % 11 == 0 {
|
||||||
|
((i + c) % 250 + 1) as u32
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
})
|
||||||
.collect()
|
.collect()
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let sparse_dir = dir.path().join("sparse");
|
let sparse_dir = dir.path().join("sparse");
|
||||||
{
|
{
|
||||||
let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap();
|
let mut b =
|
||||||
|
PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap();
|
||||||
let mut cols = Vec::new();
|
let mut cols = Vec::new();
|
||||||
let mut values = Vec::new();
|
let mut values = Vec::new();
|
||||||
for row in &rows {
|
for row in &rows {
|
||||||
@@ -229,10 +224,7 @@ fn reopen_after_close_matches_original() {
|
|||||||
fn overflow_values_roundtrip() {
|
fn overflow_values_roundtrip() {
|
||||||
// Values >= 255 exercise the PersistentCompactIntVec overflow path in
|
// Values >= 255 exercise the PersistentCompactIntVec overflow path in
|
||||||
// both the singleton and multi value streams.
|
// both the singleton and multi value streams.
|
||||||
let rows: Vec<&[u32]> = vec![
|
let rows: Vec<&[u32]> = vec![&[1000, 0, 0], &[0, 50_000, 300]];
|
||||||
&[1000, 0, 0],
|
|
||||||
&[0, 50_000, 300],
|
|
||||||
];
|
|
||||||
let (_dir, m) = make_sparse(&rows, 3);
|
let (_dir, m) = make_sparse(&rows, 3);
|
||||||
for (slot, &expected) in rows.iter().enumerate() {
|
for (slot, &expected) in rows.iter().enumerate() {
|
||||||
assert_eq!(&*m.row(slot), expected, "row {slot}");
|
assert_eq!(&*m.row(slot), expected, "row {slot}");
|
||||||
@@ -246,14 +238,13 @@ fn overflow_values_roundtrip() {
|
|||||||
/// `tests::sparse::pack_sparse_bit_matrix_transposes_columnar_directly`.
|
/// `tests::sparse::pack_sparse_bit_matrix_transposes_columnar_directly`.
|
||||||
#[test]
|
#[test]
|
||||||
fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
|
fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
|
||||||
let cols: &[&[u32]] = &[
|
let cols: &[&[u32]] = &[&[9, 0, 2, 0], &[0, 4, 0, 7], &[1, 1, 300, 1]];
|
||||||
&[9, 0, 2, 0],
|
|
||||||
&[0, 4, 0, 7],
|
|
||||||
&[1, 1, 300, 1],
|
|
||||||
];
|
|
||||||
let (dir, dense) = make_dense(cols);
|
let (dir, dense) = make_dense(cols);
|
||||||
let counts_dir = dir.path().join("counts");
|
let counts_dir = dir.path().join("counts");
|
||||||
assert!(!counts_dir.join("matrix.pcmx").exists(), "still Columnar before packing");
|
assert!(
|
||||||
|
!counts_dir.join("matrix.pcmx").exists(),
|
||||||
|
"still Columnar before packing"
|
||||||
|
);
|
||||||
|
|
||||||
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
|
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
|
||||||
let n_cols = dense.n_cols();
|
let n_cols = dense.n_cols();
|
||||||
@@ -262,9 +253,18 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
|
|||||||
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
|
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
|
||||||
|
|
||||||
assert!(counts_dir.join("singleton_values.pciv").exists());
|
assert!(counts_dir.join("singleton_values.pciv").exists());
|
||||||
assert!(!counts_dir.join("meta.json").exists(), "columnar meta.json cleaned up");
|
assert!(
|
||||||
assert!(!counts_dir.join("col_000000.pciv").exists(), "columnar column files cleaned up");
|
!counts_dir.join("meta.json").exists(),
|
||||||
assert!(!counts_dir.join("matrix.pcmx").exists(), "never materialised");
|
"columnar meta.json cleaned up"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!counts_dir.join("col_000000.pciv").exists(),
|
||||||
|
"columnar column files cleaned up"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!counts_dir.join("matrix.pcmx").exists(),
|
||||||
|
"never materialised"
|
||||||
|
);
|
||||||
|
|
||||||
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
|
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
|
||||||
assert_eq!(sparse.n(), expected_rows.len());
|
assert_eq!(sparse.n(), expected_rows.len());
|
||||||
@@ -282,10 +282,7 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() {
|
|||||||
/// it's still used as the transpose source and cleaned up afterward.
|
/// it's still used as the transpose source and cleaned up afterward.
|
||||||
#[test]
|
#[test]
|
||||||
fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
|
fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
|
||||||
let cols: &[&[u32]] = &[
|
let cols: &[&[u32]] = &[&[9, 0, 2], &[0, 4, 300]];
|
||||||
&[9, 0, 2],
|
|
||||||
&[0, 4, 300],
|
|
||||||
];
|
|
||||||
let (dir, dense) = make_dense(cols);
|
let (dir, dense) = make_dense(cols);
|
||||||
let counts_dir = dir.path().join("counts");
|
let counts_dir = dir.path().join("counts");
|
||||||
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
|
let expected_rows: Vec<Box<[u32]>> = (0..dense.n()).map(|s| dense.row(s)).collect();
|
||||||
@@ -296,7 +293,10 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
|
|||||||
assert!(counts_dir.join("matrix.pcmx").exists());
|
assert!(counts_dir.join("matrix.pcmx").exists());
|
||||||
|
|
||||||
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
|
pack_sparse_compact_int_matrix(&counts_dir).unwrap();
|
||||||
assert!(!counts_dir.join("matrix.pcmx").exists(), "packed intermediate cleaned up");
|
assert!(
|
||||||
|
!counts_dir.join("matrix.pcmx").exists(),
|
||||||
|
"packed intermediate cleaned up"
|
||||||
|
);
|
||||||
|
|
||||||
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
|
let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap();
|
||||||
assert_eq!(sparse.n_cols(), n_cols);
|
assert_eq!(sparse.n_cols(), n_cols);
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ use std::path::{Path, PathBuf};
|
|||||||
|
|
||||||
use crate::layer::utils::LAYERNAME_SUFFIX;
|
use crate::layer::utils::LAYERNAME_SUFFIX;
|
||||||
|
|
||||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
|
||||||
use obikseq::CanonicalKmer;
|
use obikseq::CanonicalKmer;
|
||||||
|
|
||||||
use crate::index::error::OKIResult;
|
use crate::index::error::OKIResult;
|
||||||
@@ -63,7 +63,7 @@ pub enum KmerLayer {
|
|||||||
Count {
|
Count {
|
||||||
dir: PathBuf,
|
dir: PathBuf,
|
||||||
id: usize,
|
id: usize,
|
||||||
layer: TypedLayer<PersistentCompactIntMatrix>,
|
layer: TypedLayer<PersistentIntMatrix>,
|
||||||
},
|
},
|
||||||
Presence {
|
Presence {
|
||||||
dir: PathBuf,
|
dir: PathBuf,
|
||||||
@@ -99,7 +99,7 @@ impl KmerLayer {
|
|||||||
already_open => return Ok(already_open),
|
already_open => return Ok(already_open),
|
||||||
};
|
};
|
||||||
if dir.join(COUNTS_DIR).exists() {
|
if dir.join(COUNTS_DIR).exists() {
|
||||||
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(&dir)?;
|
let layer = TypedLayer::<PersistentIntMatrix>::open(&dir)?;
|
||||||
Ok(KmerLayer::Count { dir, id, layer })
|
Ok(KmerLayer::Count { dir, id, layer })
|
||||||
} else {
|
} else {
|
||||||
let layer = TypedLayer::<PersistentBitMatrix>::open(&dir)?;
|
let layer = TypedLayer::<PersistentBitMatrix>::open(&dir)?;
|
||||||
@@ -221,7 +221,10 @@ impl KmerLayer {
|
|||||||
/// format-agnostic column-major fetch, delegating to whichever matrix
|
/// format-agnostic column-major fetch, delegating to whichever matrix
|
||||||
/// this layer actually holds (see `obicompactvec::PersistentBitMatrix`/
|
/// this layer actually holds (see `obicompactvec::PersistentBitMatrix`/
|
||||||
/// `PersistentCompactIntMatrix::nonzero_iter`).
|
/// `PersistentCompactIntMatrix::nonzero_iter`).
|
||||||
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
pub fn nonzero_iter<'a>(
|
||||||
|
&'a self,
|
||||||
|
slots: &'a [usize],
|
||||||
|
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
||||||
match self {
|
match self {
|
||||||
KmerLayer::Presence { layer, .. } => layer.nonzero_iter(slots),
|
KmerLayer::Presence { layer, .. } => layer.nonzero_iter(slots),
|
||||||
KmerLayer::Count { layer, .. } => layer.nonzero_iter(slots),
|
KmerLayer::Count { layer, .. } => layer.nonzero_iter(slots),
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
use obicompactvec::PersistentSparseBitMatrixBuilder;
|
use obicompactvec::PersistentSparseBitMatrixBuilder;
|
||||||
use obikseq::{set_k, Unitig};
|
use obikseq::{Unitig, set_k};
|
||||||
use obiskio::DEFAULT_BLOCK_BITS;
|
use obiskio::DEFAULT_BLOCK_BITS;
|
||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
@@ -13,7 +13,8 @@ fn write_unitigs(dir: &Path, seqs: &[&[u8]]) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn all_canonical_kmers(dir: &Path) -> Vec<CanonicalKmer> {
|
fn all_canonical_kmers(dir: &Path) -> Vec<CanonicalKmer> {
|
||||||
UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE)).unwrap()
|
UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE))
|
||||||
|
.unwrap()
|
||||||
.iter_indexed_canonical_kmers()
|
.iter_indexed_canonical_kmers()
|
||||||
.map(|(kmer, _, _)| kmer)
|
.map(|(kmer, _, _)| kmer)
|
||||||
.collect()
|
.collect()
|
||||||
@@ -34,13 +35,17 @@ fn canonical_kmer_iter_matches_reader() {
|
|||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
write_unitigs(dir.path(), &[b"AAAACGT", b"TTTTGCA"]);
|
write_unitigs(dir.path(), &[b"AAAACGT", b"TTTTGCA"]);
|
||||||
|
|
||||||
let from_iter: Vec<CanonicalKmer> = obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE))
|
let from_iter: Vec<CanonicalKmer> =
|
||||||
|
obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.collect();
|
.collect();
|
||||||
let from_reader: Vec<CanonicalKmer> = all_canonical_kmers(dir.path());
|
let from_reader: Vec<CanonicalKmer> = all_canonical_kmers(dir.path());
|
||||||
|
|
||||||
assert_eq!(from_iter.len(), from_reader.len(), "different kmer counts");
|
assert_eq!(from_iter.len(), from_reader.len(), "different kmer counts");
|
||||||
assert_eq!(from_iter, from_reader, "CanonicalKmerIter and UnitigFileReader disagree");
|
assert_eq!(
|
||||||
|
from_iter, from_reader,
|
||||||
|
"CanonicalKmerIter and UnitigFileReader disagree"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Generic `TypedLayer<D>` over dense vs. sparse presence storage ───────────────
|
// ── Generic `TypedLayer<D>` over dense vs. sparse presence storage ───────────────
|
||||||
@@ -64,9 +69,14 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
|||||||
// kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be
|
// kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be
|
||||||
// biologically meaningful, just the same on both sides of the
|
// biologically meaningful, just the same on both sides of the
|
||||||
// dense/sparse comparison below.
|
// dense/sparse comparison below.
|
||||||
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
|
TypedLayer::<PersistentBitMatrix>::build_presence(
|
||||||
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
dir.path(),
|
||||||
}).unwrap();
|
DEFAULT_BLOCK_BITS,
|
||||||
|
&mode,
|
||||||
|
n_genomes,
|
||||||
|
|kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
|
let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
|
||||||
assert!(dense_layer.n_cols() >= 1);
|
assert!(dense_layer.n_cols() >= 1);
|
||||||
@@ -76,7 +86,10 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
|||||||
// same directory, distinct filenames — see
|
// same directory, distinct filenames — see
|
||||||
// `DevDocMD/architecture/siblings.md`'s sparse-matrix section).
|
// `DevDocMD/architecture/siblings.md`'s sparse-matrix section).
|
||||||
let dense_matrix = obicompactvec::PersistentBitMatrix::open(dir.path()).unwrap();
|
let dense_matrix = obicompactvec::PersistentBitMatrix::open(dir.path()).unwrap();
|
||||||
PersistentSparseBitMatrixBuilder::build_from_dense(&dense_matrix, &dir.path().join(PRESENCE_DIR))
|
PersistentSparseBitMatrixBuilder::build_from_dense(
|
||||||
|
&dense_matrix,
|
||||||
|
&dir.path().join(PRESENCE_DIR),
|
||||||
|
)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.close()
|
.close()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -88,7 +101,10 @@ fn presence_layer_generic_over_sparse_matches_dense() {
|
|||||||
let n = dense_layer.n();
|
let n = dense_layer.n();
|
||||||
assert_eq!(n, sparse_layer.n());
|
assert_eq!(n, sparse_layer.n());
|
||||||
let slots: Vec<usize> = (0..n).collect();
|
let slots: Vec<usize> = (0..n).collect();
|
||||||
assert_eq!(dense_layer.sub_matrix(&slots), sparse_layer.sub_matrix(&slots));
|
assert_eq!(
|
||||||
|
dense_layer.sub_matrix(&slots),
|
||||||
|
sparse_layer.sub_matrix(&slots)
|
||||||
|
);
|
||||||
|
|
||||||
// The matrix-agnostic, MPHF-only surface (from the generic
|
// The matrix-agnostic, MPHF-only surface (from the generic
|
||||||
// `impl<D: LayerData> TypedLayer<D>`) must also agree: same kmer set,
|
// `impl<D: LayerData> TypedLayer<D>`) must also agree: same kmer set,
|
||||||
@@ -124,15 +140,25 @@ fn count_layer_transparently_reads_sparse_after_pack() {
|
|||||||
// Deterministic, arbitrary count function — same shape as the presence
|
// Deterministic, arbitrary count function — same shape as the presence
|
||||||
// test's arbitrary predicate, just producing a small count instead of a
|
// test's arbitrary predicate, just producing a small count instead of a
|
||||||
// bool.
|
// bool.
|
||||||
TypedLayer::<()>::build_with_matrix(dir.path(), DEFAULT_BLOCK_BITS, &mode, false, n_genomes, |kmer| {
|
TypedLayer::<()>::build_with_matrix(
|
||||||
|
dir.path(),
|
||||||
|
DEFAULT_BLOCK_BITS,
|
||||||
|
&mode,
|
||||||
|
false,
|
||||||
|
n_genomes,
|
||||||
|
|kmer| {
|
||||||
(0..n_genomes)
|
(0..n_genomes)
|
||||||
.map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32)
|
.map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32)
|
||||||
.collect()
|
.collect()
|
||||||
})
|
},
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let dense_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
|
let dense_layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
|
||||||
assert_eq!(dense_layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
assert_eq!(
|
||||||
|
dense_layer.storage_kind(),
|
||||||
|
obicompactvec::StorageKind::Columnar
|
||||||
|
);
|
||||||
let n = dense_layer.n();
|
let n = dense_layer.n();
|
||||||
let slots: Vec<usize> = (0..n).collect();
|
let slots: Vec<usize> = (0..n).collect();
|
||||||
let dense_sub = dense_layer.sub_matrix(&slots);
|
let dense_sub = dense_layer.sub_matrix(&slots);
|
||||||
@@ -146,8 +172,11 @@ fn count_layer_transparently_reads_sparse_after_pack() {
|
|||||||
// in production.
|
// in production.
|
||||||
obicompactvec::pack_sparse_compact_int_matrix(&dir.path().join(COUNTS_DIR)).unwrap();
|
obicompactvec::pack_sparse_compact_int_matrix(&dir.path().join(COUNTS_DIR)).unwrap();
|
||||||
|
|
||||||
let sparse_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
|
let sparse_layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
|
||||||
assert_eq!(sparse_layer.storage_kind(), obicompactvec::StorageKind::Sparse);
|
assert_eq!(
|
||||||
|
sparse_layer.storage_kind(),
|
||||||
|
obicompactvec::StorageKind::Sparse
|
||||||
|
);
|
||||||
assert_eq!(sparse_layer.n(), n);
|
assert_eq!(sparse_layer.n(), n);
|
||||||
assert_eq!(sparse_layer.n_cols(), dense_layer.n_cols());
|
assert_eq!(sparse_layer.n_cols(), dense_layer.n_cols());
|
||||||
assert_eq!(sparse_layer.sub_matrix(&slots), dense_sub);
|
assert_eq!(sparse_layer.sub_matrix(&slots), dense_sub);
|
||||||
@@ -167,13 +196,20 @@ fn count_layer_reports_count_content_and_columnar_storage() {
|
|||||||
set_k(4);
|
set_k(4);
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
write_unitigs(dir.path(), &[b"AAAACGT"]);
|
write_unitigs(dir.path(), &[b"AAAACGT"]);
|
||||||
TypedLayer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
|
TypedLayer::<PersistentIntMatrix>::build(dir.pat
|
||||||
.unwrap();
|
h(), DEFAUL
|
||||||
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
|
T_BLOCK_BITS, &Index
|
||||||
|
Mode::Exact, |_| 1)
|
||||||
|
,
|
||||||
|
).unwrap();
|
||||||
|
let layer = TypedLayer::<PersistentIntMatrix>::open(dir.path()).unwrap();
|
||||||
|
|
||||||
assert_eq!(layer.content(), LayerContent::Count);
|
assert_eq!(layer.content(), LayerContent::Count);
|
||||||
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
|
||||||
assert_eq!(layer.evidence_kind(), crate::layer::mphf_layer::EvidenceKind::Exact);
|
assert_eq!(
|
||||||
|
layer.evidence_kind(),
|
||||||
|
crate::layer::mphf_layer::EvidenceKind::Exact
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -181,9 +217,14 @@ fn presence_layer_reports_presence_content_and_columnar_storage() {
|
|||||||
set_k(4);
|
set_k(4);
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
|
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
|
||||||
TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
|
TypedLayer::<PersistentBitMatrix>::build_presence(
|
||||||
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
|
dir.path(),
|
||||||
}).unwrap();
|
DEFAULT_BLOCK_BITS,
|
||||||
|
&IndexMode::Exact,
|
||||||
|
2,
|
||||||
|
|kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
|
let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
|
||||||
|
|
||||||
assert_eq!(layer.content(), LayerContent::Presence);
|
assert_eq!(layer.content(), LayerContent::Presence);
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// use crate::layer::utils::layer_dir;
|
// use crate::layer::utils::layer_dir;
|
||||||
use obicompactvec::{
|
use obicompactvec::{
|
||||||
BinaryMatrix, ColBuilder, MatrixBuilder, PersistentBitMatrix, PersistentBitMatrixBuilder,
|
BinaryMatrix, ColBuilder, MatrixBuilder, PersistentBitMatrix, PersistentBitMatrixBuilder,
|
||||||
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentSparseBitMatrix,
|
PersistentCompactIntMatrixBuilder, PersistentIntMatrix, PersistentSparseBitMatrix,
|
||||||
};
|
};
|
||||||
use obikseq::CanonicalKmer;
|
use obikseq::CanonicalKmer;
|
||||||
use obiskio::{UnitigFileReader, UnitigFileWriter};
|
use obiskio::{UnitigFileReader, UnitigFileWriter};
|
||||||
@@ -47,10 +47,10 @@ impl LayerData for () {
|
|||||||
fn read(&self, _slot: usize) {}
|
fn read(&self, _slot: usize) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LayerData for PersistentCompactIntMatrix {
|
impl LayerData for PersistentIntMatrix {
|
||||||
type Item = Box<[u32]>;
|
type Item = Box<[u32]>;
|
||||||
fn open(layer_dir: &Path) -> OKIResult<Self> {
|
fn open(layer_dir: &Path) -> OKIResult<Self> {
|
||||||
PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)
|
PersistentIntMatrix::open(layer_dir).map_err(OKIError::Io)
|
||||||
}
|
}
|
||||||
fn read(&self, slot: usize) -> Box<[u32]> {
|
fn read(&self, slot: usize) -> Box<[u32]> {
|
||||||
self.row(slot)
|
self.row(slot)
|
||||||
@@ -112,7 +112,7 @@ pub trait HasLayerContent {
|
|||||||
const CONTENT: LayerContent;
|
const CONTENT: LayerContent;
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HasLayerContent for PersistentCompactIntMatrix {
|
impl HasLayerContent for PersistentIntMatrix {
|
||||||
const CONTENT: LayerContent = LayerContent::Count;
|
const CONTENT: LayerContent = LayerContent::Count;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -135,9 +135,9 @@ pub trait HasStorageKind {
|
|||||||
fn storage_kind(&self) -> obicompactvec::StorageKind;
|
fn storage_kind(&self) -> obicompactvec::StorageKind;
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HasStorageKind for PersistentCompactIntMatrix {
|
impl HasStorageKind for PersistentIntMatrix {
|
||||||
fn storage_kind(&self) -> obicompactvec::StorageKind {
|
fn storage_kind(&self) -> obicompactvec::StorageKind {
|
||||||
PersistentCompactIntMatrix::storage_kind(self)
|
PersistentIntMatrix::storage_kind(self)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -342,7 +342,7 @@ impl TypedLayer<()> {
|
|||||||
|
|
||||||
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
|
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
impl TypedLayer<PersistentCompactIntMatrix> {
|
impl TypedLayer<PersistentIntMatrix> {
|
||||||
pub fn build(
|
pub fn build(
|
||||||
out_dir: &Path,
|
out_dir: &Path,
|
||||||
block_bits: u8,
|
block_bits: u8,
|
||||||
@@ -377,12 +377,12 @@ impl TypedLayer<PersistentCompactIntMatrix> {
|
|||||||
|
|
||||||
// ── Mode 2 — count matrix column append ──────────────────────────────────────
|
// ── Mode 2 — count matrix column append ──────────────────────────────────────
|
||||||
|
|
||||||
impl TypedLayer<PersistentCompactIntMatrix> {
|
impl TypedLayer<PersistentIntMatrix> {
|
||||||
pub fn append_genome_column(
|
pub fn append_genome_column(
|
||||||
layer_dir: &Path,
|
layer_dir: &Path,
|
||||||
value_of: impl Fn(usize) -> u32,
|
value_of: impl Fn(usize) -> u32,
|
||||||
) -> OKIResult<()> {
|
) -> OKIResult<()> {
|
||||||
PersistentCompactIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of)
|
PersistentIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of)
|
||||||
.map_err(OKIError::Io)
|
.map_err(OKIError::Io)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -418,7 +418,10 @@ impl TypedLayer<PersistentCompactIntMatrix> {
|
|||||||
|
|
||||||
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
|
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
|
||||||
/// delegates to `PersistentCompactIntMatrix::nonzero_iter`.
|
/// delegates to `PersistentCompactIntMatrix::nonzero_iter`.
|
||||||
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
pub fn nonzero_iter<'a>(
|
||||||
|
&'a self,
|
||||||
|
slots: &'a [usize],
|
||||||
|
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
||||||
self.data.nonzero_iter(slots)
|
self.data.nonzero_iter(slots)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -477,7 +480,10 @@ impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix + obicompactvec::ColumnWeig
|
|||||||
impl TypedLayer<PersistentBitMatrix> {
|
impl TypedLayer<PersistentBitMatrix> {
|
||||||
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
|
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
|
||||||
/// delegates to `PersistentBitMatrix::nonzero_iter`.
|
/// delegates to `PersistentBitMatrix::nonzero_iter`.
|
||||||
pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
pub fn nonzero_iter<'a>(
|
||||||
|
&'a self,
|
||||||
|
slots: &'a [usize],
|
||||||
|
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
|
||||||
self.data.nonzero_iter(slots)
|
self.data.nonzero_iter(slots)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -24,14 +24,14 @@ use std::fs;
|
|||||||
use std::io;
|
use std::io;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
|
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
|
||||||
use cacheline_ef::{CachelineEf, CachelineEfVec};
|
use cacheline_ef::{CachelineEf, CachelineEfVec};
|
||||||
use epserde::prelude::*;
|
use epserde::prelude::*;
|
||||||
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
|
use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec};
|
||||||
use obidebruinj::GraphDeBruijn;
|
use obidebruinj::GraphDeBruijn;
|
||||||
use obikindex::layer::IndexMode;
|
use obikindex::layer::IndexMode;
|
||||||
use obikindex::layer::{KmerLayer, TypedLayer};
|
use obikindex::layer::{KmerLayer, TypedLayer};
|
||||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||||
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
|
|
||||||
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
||||||
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
||||||
|
|
||||||
@@ -139,7 +139,9 @@ impl PrivateBuilder for KmerIndex {
|
|||||||
mode: &IndexMode,
|
mode: &IndexMode,
|
||||||
block_bits: u8,
|
block_bits: u8,
|
||||||
) -> Result<usize, SKError> {
|
) -> Result<usize, SKError> {
|
||||||
let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?;
|
let layer0 = self
|
||||||
|
.layer0(i)
|
||||||
|
.map_err(|e| io::Error::other(e.to_string()))?;
|
||||||
let layer0_dir = layer0.dir();
|
let layer0_dir = layer0.dir();
|
||||||
let dedup_path = layer0.dereplicated_superkmers_path();
|
let dedup_path = layer0.dereplicated_superkmers_path();
|
||||||
if !dedup_path.exists() {
|
if !dedup_path.exists() {
|
||||||
@@ -193,15 +195,12 @@ impl PrivateBuilder for KmerIndex {
|
|||||||
let n_kmers = if with_counts {
|
let n_kmers = if with_counts {
|
||||||
let n = write_graph_as_unitigs(g, layer0_dir)
|
let n = write_graph_as_unitigs(g, layer0_dir)
|
||||||
.map_err(|e| io::Error::other(e.to_string()))?;
|
.map_err(|e| io::Error::other(e.to_string()))?;
|
||||||
TypedLayer::<PersistentCompactIntMatrix>::build(
|
TypedLayer::<PersistentIntMatrix>::build(layer0_dir, block_bits, mode, |kmer| {
|
||||||
layer0_dir,
|
match (&mphf1_opt, &counts1_opt) {
|
||||||
block_bits,
|
|
||||||
mode,
|
|
||||||
|kmer| match (&mphf1_opt, &counts1_opt) {
|
|
||||||
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
|
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
|
||||||
_ => 1,
|
_ => 1,
|
||||||
},
|
}
|
||||||
)
|
})
|
||||||
.map_err(|e| io::Error::other(e.to_string()))?;
|
.map_err(|e| io::Error::other(e.to_string()))?;
|
||||||
n
|
n
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ fn presence(mode: MergeMode) -> bool {
|
|||||||
mod matrix_builder_tests {
|
mod matrix_builder_tests {
|
||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
|
||||||
|
|
||||||
use super::{ColBuilder, MatrixBuilder};
|
use super::{ColBuilder, MatrixBuilder};
|
||||||
|
|
||||||
@@ -90,7 +90,7 @@ mod matrix_builder_tests {
|
|||||||
col.close().unwrap();
|
col.close().unwrap();
|
||||||
mb.close().unwrap();
|
mb.close().unwrap();
|
||||||
|
|
||||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
let m = PersistentIntMatrix::open(dir.path()).unwrap();
|
||||||
assert_eq!(m.n_cols(), 2);
|
assert_eq!(m.n_cols(), 2);
|
||||||
assert_eq!(&*m.row(0), &[0u32, 7]);
|
assert_eq!(&*m.row(0), &[0u32, 7]);
|
||||||
assert_eq!(&*m.row(1), &[0u32, 42]);
|
assert_eq!(&*m.row(1), &[0u32, 42]);
|
||||||
@@ -422,8 +422,13 @@ pub(crate) fn merge_partition(
|
|||||||
move |data: Pass2Data,
|
move |data: Pass2Data,
|
||||||
push: &PipelineSender<Result<Pass2Data, PipelineError>>,
|
push: &PipelineSender<Result<Pass2Data, PipelineError>>,
|
||||||
delta: &PipelineSender<isize>| {
|
delta: &PipelineSender<isize>| {
|
||||||
if let Pass2Data::SrcLayer((col_offset, src_n, unitigs_path, src_layer, _guard)) =
|
if let Pass2Data::SrcLayer((
|
||||||
data
|
col_offset,
|
||||||
|
src_n,
|
||||||
|
unitigs_path,
|
||||||
|
src_layer,
|
||||||
|
_guard,
|
||||||
|
)) = data
|
||||||
{
|
{
|
||||||
// _guard dropped at end of block, releasing the slot.
|
// _guard dropped at end of block, releasing the slot.
|
||||||
// `src_layer` (MPHF + matrix) was already opened up
|
// `src_layer` (MPHF + matrix) was already opened up
|
||||||
@@ -455,7 +460,9 @@ pub(crate) fn merge_partition(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !batch.is_empty() {
|
if !batch.is_empty() {
|
||||||
push.send(Ok(Pass2Data::RawBatch((col_offset, src_n, src_layer, batch))))
|
push.send(Ok(Pass2Data::RawBatch((
|
||||||
|
col_offset, src_n, src_layer, batch,
|
||||||
|
))))
|
||||||
.ok();
|
.ok();
|
||||||
count += 1;
|
count += 1;
|
||||||
}
|
}
|
||||||
@@ -480,8 +487,9 @@ pub(crate) fn merge_partition(
|
|||||||
// per kmer — matches the underlying matrix's own
|
// per kmer — matches the underlying matrix's own
|
||||||
// column-major layout.
|
// column-major layout.
|
||||||
let slots = src_layer.hash_batch(&kmers);
|
let slots = src_layer.hash_batch(&kmers);
|
||||||
let mut cols: Vec<Vec<u32>> =
|
let mut cols: Vec<Vec<u32>> = (0..src_n)
|
||||||
(0..src_n).map(|_| Vec::with_capacity(kmers.len())).collect();
|
.map(|_| Vec::with_capacity(kmers.len()))
|
||||||
|
.collect();
|
||||||
src_layer.fill_sub_matrix(&slots, &mut cols);
|
src_layer.fill_sub_matrix(&slots, &mut cols);
|
||||||
|
|
||||||
// Membership against dst, grouped by dst layer
|
// Membership against dst, grouped by dst layer
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
|
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix};
|
||||||
|
use obikidxcache::LayeredStore;
|
||||||
use obikindex::OKIResult;
|
use obikindex::OKIResult;
|
||||||
use obikindex::layer::open_data;
|
use obikindex::layer::open_data;
|
||||||
use obikidxcache::LayeredStore;
|
|
||||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
|
||||||
|
|
||||||
use obikindex::load_meta;
|
|
||||||
use obikindex::KmerIndex;
|
use obikindex::KmerIndex;
|
||||||
|
use obikindex::load_meta;
|
||||||
|
|
||||||
impl KmerIndex {
|
impl KmerIndex {
|
||||||
/// Open all count matrices for partition `part`, one per layer.
|
/// Open all count matrices for partition `part`, one per layer.
|
||||||
/// Layers without a `counts/` directory are skipped.
|
/// Layers without a `counts/` directory are skipped.
|
||||||
pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentCompactIntMatrix>> {
|
pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentIntMatrix>> {
|
||||||
let index_dir = self.index_dir(part);
|
let index_dir = self.index_dir(part);
|
||||||
if !index_dir.exists() {
|
if !index_dir.exists() {
|
||||||
return Ok(LayeredStore::new(vec![]));
|
return Ok(LayeredStore::new(vec![]));
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ use std::io;
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use obicompactvec::{
|
use obicompactvec::{
|
||||||
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix,
|
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix, PersistentIntMatrix, TempBitVec,
|
||||||
PersistentCompactIntMatrix, TempBitVec, TempCompactIntVec,
|
TempCompactIntVec,
|
||||||
};
|
};
|
||||||
use obikindex::layer::{KmerLayer, LayerContent};
|
use obikindex::layer::{KmerLayer, LayerContent};
|
||||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||||
@@ -45,7 +45,9 @@ impl AggOp {
|
|||||||
"sum" => Ok(AggOp::Sum),
|
"sum" => Ok(AggOp::Sum),
|
||||||
"min" => Ok(AggOp::Min),
|
"min" => Ok(AggOp::Min),
|
||||||
"max" => Ok(AggOp::Max),
|
"max" => Ok(AggOp::Max),
|
||||||
other => Err(format!("unknown aggregation operator: {other}; valid: any, all, none, sum, min, max")),
|
other => Err(format!(
|
||||||
|
"unknown aggregation operator: {other}; valid: any, all, none, sum, min, max"
|
||||||
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,7 +77,11 @@ enum AggResult {
|
|||||||
Int(TempCompactIntVec),
|
Int(TempCompactIntVec),
|
||||||
}
|
}
|
||||||
|
|
||||||
fn compute_group(mat: &dyn MatrixGroupOps, spec: &OutputCol, threshold: u32) -> io::Result<AggResult> {
|
fn compute_group(
|
||||||
|
mat: &dyn MatrixGroupOps,
|
||||||
|
spec: &OutputCol,
|
||||||
|
threshold: u32,
|
||||||
|
) -> io::Result<AggResult> {
|
||||||
let g = ColGroup::new(spec.label.clone(), spec.indices.clone());
|
let g = ColGroup::new(spec.label.clone(), spec.indices.clone());
|
||||||
Ok(match spec.op {
|
Ok(match spec.op {
|
||||||
AggOp::Any => AggResult::Bit(mat.partial_group_any(&g, threshold)?),
|
AggOp::Any => AggResult::Bit(mat.partial_group_any(&g, threshold)?),
|
||||||
@@ -145,18 +151,23 @@ pub(crate) fn select_partition(
|
|||||||
|
|
||||||
let group_mat: Box<dyn MatrixGroupOps> = match src_layer.content() {
|
let group_mat: Box<dyn MatrixGroupOps> = match src_layer.content() {
|
||||||
LayerContent::Count => {
|
LayerContent::Count => {
|
||||||
Box::new(PersistentCompactIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
|
Box::new(PersistentIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
|
||||||
}
|
}
|
||||||
LayerContent::Presence => {
|
LayerContent::Presence => {
|
||||||
Box::new(PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
|
Box::new(PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let data_subdir = if output_presence { "presence" } else { "counts" };
|
let data_subdir = if output_presence {
|
||||||
|
"presence"
|
||||||
|
} else {
|
||||||
|
"counts"
|
||||||
|
};
|
||||||
let data_dir = dst_layer_dir.join(data_subdir);
|
let data_dir = dst_layer_dir.join(data_subdir);
|
||||||
fs::create_dir_all(&data_dir).map_err(OKIError::Io)?;
|
fs::create_dir_all(&data_dir).map_err(OKIError::Io)?;
|
||||||
|
|
||||||
let mut builder = MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?;
|
let mut builder =
|
||||||
|
MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?;
|
||||||
for spec in specs {
|
for spec in specs {
|
||||||
let r = compute_group(group_mat.as_ref(), spec, threshold).map_err(OKIError::Io)?;
|
let r = compute_group(group_mat.as_ref(), spec, threshold).map_err(OKIError::Io)?;
|
||||||
add_result(&mut builder, r).map_err(OKIError::Io)?;
|
add_result(&mut builder, r).map_err(OKIError::Io)?;
|
||||||
|
|||||||
Reference in New Issue
Block a user