Implement MPHF migration tool and sparse matrix permutation logic

Refactors sparse matrix handling to implement row permutation logic and introduces a standalone tool for migrating indexes from MPHF 1.x to 2.x formats.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:12:51 +02:00
1 parent d686d39506
commit 8b87078478
24 files changed
+3982 -19

No files matched your search

+25
View File
@@ -514,6 +514,31 @@ impl PersistentSparseBitMatrixBuilder {
}
Ok(builder)
}
/// Builds the row-permuted copy of `src` into `dir`: row `s` of the
/// result is row `new_to_old[s]` of `src`. `new_to_old` must have
/// `src.n()` entries and be a permutation of `0..src.n()` — the caller
/// proves that, this only reads `src` at those slots.
///
/// Needed whenever the slot numbering of a layer changes (its MPHF is
/// rebuilt) while the data attached to each k-mer must follow it.
/// Rows are read at arbitrary slots of `src`, so convert a dense source
/// to sparse first (`build_from_dense`) rather than permuting it
/// cell by cell.
pub fn build_permuted(
src: &PersistentSparseBitMatrix,
new_to_old: &[u32],
dir: &Path,
) -> io::Result<Self> {
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
let mut genomes: Vec<u32> = Vec::new();
for &old in new_to_old {
genomes.clear();
src.for_each_genome_in_row(old as usize, |g| genomes.push(g as u32));
builder.push_row(&genomes);
}
Ok(builder)
}
}
// ── Trait impls ───────────────────────────────────────────────────────────────
+23
View File
@@ -548,6 +548,29 @@ impl PersistentSparseCompactIntMatrixBuilder {
Ok(builder)
}
/// Row-permuted copy of `src`: row `s` of the result is row
/// `new_to_old[s]` of `src`. Same contract as
/// [`PersistentSparseBitMatrixBuilder::build_permuted`].
pub fn build_permuted(
src: &PersistentSparseCompactIntMatrix,
new_to_old: &[u32],
dir: &Path,
) -> io::Result<Self> {
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
let mut cols: Vec<u32> = Vec::new();
let mut vals: Vec<u32> = Vec::new();
for &old in new_to_old {
cols.clear();
vals.clear();
src.for_each_cell_in_row(old as usize, |c, v| {
cols.push(c as u32);
vals.push(v);
});
builder.push_row(&cols, &vals);
}
Ok(builder)
}
pub fn close(self) -> io::Result<()> {
let Self {
support,
+28
View File
@@ -415,3 +415,31 @@ fn partial_jaccard_and_hamming_match_dense_including_diagonal() {
drop(dir_dense);
drop(dir_sparse);
}
#[test]
fn build_permuted_follows_the_permutation() {
// Mix of singleton and multi rows, including a repeated multi set, so the
// dictionary dedup is exercised on the permuted order too.
let rows: Vec<&[bool]> = vec![
&[true, false, false, false],
&[false, false, true, false],
&[true, true, false, false],
&[false, false, true, true],
&[true, true, false, false],
];
let (_dir, src) = make_sparse(&rows, 4);
let new_to_old: Vec<u32> = vec![3, 0, 4, 1, 2];
let out = tempdir().unwrap();
let permuted =
PersistentSparseBitMatrixBuilder::build_permuted(&src, &new_to_old, &out.path().join("p"))
.unwrap()
.finish()
.unwrap();
assert_eq!(permuted.n(), src.n());
assert_eq!(permuted.n_cols(), src.n_cols());
for (new, &old) in new_to_old.iter().enumerate() {
assert_eq!(row_as_bool(&permuted, new), row_as_bool(&src, old as usize), "new row {new}");
}
}
@@ -304,3 +304,32 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
assert_eq!(&*sparse.row(slot), &**expected, "slot {slot}");
}
}
#[test]
fn build_permuted_follows_the_permutation() {
// Rows sharing a support with different values, plus singletons.
let rows: Vec<&[u32]> = vec![
&[7, 0, 0, 0],
&[0, 0, 3, 0],
&[2, 5, 0, 0],
&[9, 1, 0, 0],
&[0, 0, 4, 6],
];
let (_dir, src) = make_sparse(&rows, 4);
let new_to_old: Vec<u32> = vec![4, 2, 0, 3, 1];
let out = tempdir().unwrap();
let permuted = PersistentSparseCompactIntMatrixBuilder::build_permuted(
&src,
&new_to_old,
&out.path().join("p"),
)
.unwrap()
.finish()
.unwrap();
assert_eq!(permuted.n(), src.n());
for (new, &old) in new_to_old.iter().enumerate() {
assert_eq!(permuted.row(new), src.row(old as usize), "new row {new}");
}
}