Implement MPHF migration tool and sparse matrix permutation logic

Refactors sparse matrix handling to implement row permutation logic and introduces a standalone tool for migrating indexes from MPHF 1.x to 2.x formats.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:12:51 +02:00
1 parent d686d39506
commit 8b87078478
24 files changed
+3982 -19

No files matched your search

-2
View File
@@ -1528,7 +1528,6 @@ dependencies = [
name = "obikindexer"
version = "0.1.0"
dependencies = [
"cacheline-ef",
"epserde",
"memmap2",
"niffler",
@@ -1543,7 +1542,6 @@ dependencies = [
"obiskbuilder",
"obiskio",
"obisys",
"ptr_hash",
"rayon",
"serde",
"serde_json",
+25
View File
@@ -514,6 +514,31 @@ impl PersistentSparseBitMatrixBuilder {
}
Ok(builder)
}
/// Builds the row-permuted copy of `src` into `dir`: row `s` of the
/// result is row `new_to_old[s]` of `src`. `new_to_old` must have
/// `src.n()` entries and be a permutation of `0..src.n()` — the caller
/// proves that, this only reads `src` at those slots.
///
/// Needed whenever the slot numbering of a layer changes (its MPHF is
/// rebuilt) while the data attached to each k-mer must follow it.
/// Rows are read at arbitrary slots of `src`, so convert a dense source
/// to sparse first (`build_from_dense`) rather than permuting it
/// cell by cell.
pub fn build_permuted(
src: &PersistentSparseBitMatrix,
new_to_old: &[u32],
dir: &Path,
) -> io::Result<Self> {
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
let mut genomes: Vec<u32> = Vec::new();
for &old in new_to_old {
genomes.clear();
src.for_each_genome_in_row(old as usize, |g| genomes.push(g as u32));
builder.push_row(&genomes);
}
Ok(builder)
}
}
// ── Trait impls ───────────────────────────────────────────────────────────────
+23
View File
@@ -548,6 +548,29 @@ impl PersistentSparseCompactIntMatrixBuilder {
Ok(builder)
}
/// Row-permuted copy of `src`: row `s` of the result is row
/// `new_to_old[s]` of `src`. Same contract as
/// [`PersistentSparseBitMatrixBuilder::build_permuted`].
pub fn build_permuted(
src: &PersistentSparseCompactIntMatrix,
new_to_old: &[u32],
dir: &Path,
) -> io::Result<Self> {
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
let mut cols: Vec<u32> = Vec::new();
let mut vals: Vec<u32> = Vec::new();
for &old in new_to_old {
cols.clear();
vals.clear();
src.for_each_cell_in_row(old as usize, |c, v| {
cols.push(c as u32);
vals.push(v);
});
builder.push_row(&cols, &vals);
}
Ok(builder)
}
pub fn close(self) -> io::Result<()> {
let Self {
support,
+28
View File
@@ -415,3 +415,31 @@ fn partial_jaccard_and_hamming_match_dense_including_diagonal() {
drop(dir_dense);
drop(dir_sparse);
}
#[test]
fn build_permuted_follows_the_permutation() {
// Mix of singleton and multi rows, including a repeated multi set, so the
// dictionary dedup is exercised on the permuted order too.
let rows: Vec<&[bool]> = vec![
&[true, false, false, false],
&[false, false, true, false],
&[true, true, false, false],
&[false, false, true, true],
&[true, true, false, false],
];
let (_dir, src) = make_sparse(&rows, 4);
let new_to_old: Vec<u32> = vec![3, 0, 4, 1, 2];
let out = tempdir().unwrap();
let permuted =
PersistentSparseBitMatrixBuilder::build_permuted(&src, &new_to_old, &out.path().join("p"))
.unwrap()
.finish()
.unwrap();
assert_eq!(permuted.n(), src.n());
assert_eq!(permuted.n_cols(), src.n_cols());
for (new, &old) in new_to_old.iter().enumerate() {
assert_eq!(row_as_bool(&permuted, new), row_as_bool(&src, old as usize), "new row {new}");
}
}
@@ -304,3 +304,32 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
assert_eq!(&*sparse.row(slot), &**expected, "slot {slot}");
}
}
#[test]
fn build_permuted_follows_the_permutation() {
// Rows sharing a support with different values, plus singletons.
let rows: Vec<&[u32]> = vec![
&[7, 0, 0, 0],
&[0, 0, 3, 0],
&[2, 5, 0, 0],
&[9, 1, 0, 0],
&[0, 0, 4, 6],
];
let (_dir, src) = make_sparse(&rows, 4);
let new_to_old: Vec<u32> = vec![4, 2, 0, 3, 1];
let out = tempdir().unwrap();
let permuted = PersistentSparseCompactIntMatrixBuilder::build_permuted(
&src,
&new_to_old,
&out.path().join("p"),
)
.unwrap()
.finish()
.unwrap();
assert_eq!(permuted.n(), src.n());
for (new, &old) in new_to_old.iter().enumerate() {
assert_eq!(permuted.row(new), src.row(old as usize), "new row {new}");
}
}
+8
View File
@@ -210,6 +210,14 @@ impl KmerIndex {
crate::partition::index_dir(&self.root_path, i)
}
/// Path of layer `j` of partition `i` — pure path computation, nothing
/// is opened (unlike [`layer`](Self::layer)/`layer_dir`, which open the
/// layer's MPHF). For maintenance tools that must reach a layer's files
/// without going through a format they can't read yet.
pub fn layer_path(&self, i: usize, j: usize) -> PathBuf {
crate::layer::utils::layer_dir(&self.index_dir(i), j)
}
/// Number of layers in partition `i` — a disk scan, not a metadata read
/// (see [`KmerPartition::n_layers`](crate::partition::KmerPartition::n_layers)).
pub fn n_layers(&self, i: usize) -> OKIResult<usize> {
+3 -1
View File
@@ -9,5 +9,7 @@ pub(crate) mod utils;
pub use crate::index::error::{OKIError, OKIResult};
pub use content_layer::KmerLayer;
pub use meta::IndexMode;
pub use mphf_layer::{EvidenceKind, KmerBatchIter, KmerIter, MphfLayer, MphfOnly, UnitigIter};
pub use mphf_layer::{
EvidenceKind, KmerBatchIter, KmerIter, Mphf, MphfLayer, MphfOnly, UnitigIter, mphf_params,
};
pub use typed_layer::{HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer};
+9 -3
View File
@@ -23,7 +23,13 @@ pub(crate) const FINGERPRINT_FILE: &str = "fingerprint.bin";
/// The layer MPHF: `Xxh3Int` hasher, `CubicEps` buckets, multi-part (`SINGLE_PART = false`),
/// minimal (`REMAP = true`). Built owned (construction + store); queried through
/// `MemCase<Mphf>::uncase()`, the ε-copy view into the memory-mapped file.
pub(crate) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
pub type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
/// Construction parameters of [`Mphf`] — the one place they are chosen
/// (`lambda = 3.5`, `alpha = 0.99`, `CubicEps`, ~2.4 bits/key).
pub fn mphf_params() -> PtrHashParams<CubicEps> {
PtrHashParams::<CubicEps>::default_balanced()
}
// ── LayerEvidence ─────────────────────────────────────────────────────────────
@@ -550,7 +556,7 @@ impl MphfLayer {
// ── Empty layer ───────────────────────────────────────────────────────
if n == 0 {
let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::<CubicEps>::default_balanced())
let mphf: Mphf = Mphf::try_new(&[] as &[u64], mphf_params())
.ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?;
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
@@ -574,7 +580,7 @@ impl MphfLayer {
let mphf: Mphf = Mphf::new_from_par_iter(
n,
keys.map(|k| k.raw()).par_bridge(),
PtrHashParams::<CubicEps>::default_balanced(),
mphf_params(),
);
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
-2
View File
@@ -14,9 +14,7 @@ obicompactvec = { path = "../obicompactvec" }
obipipeline = { path = "../obipipeline" }
obiread = { path = "../obiread" }
obiskbuilder = { path = "../obiskbuilder" }
cacheline-ef = "2.0"
epserde = "0.13"
ptr_hash = { version = "2.1", features = ["epserde"] }
niffler = "3.0.0"
memmap2 = "0.9.10"
rayon = "1"
@@ -3,18 +3,15 @@ use std::fs;
use std::io;
use std::path::Path;
use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::ser::Serialize as EpSerialize;
use memmap2::Mmap;
use obicompactvec::PersistentCompactIntVecBuilder;
use obikindex::layer::{Mphf, mphf_params};
use obiskio::{SKFileReader, SKResult};
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int};
use tracing::debug;
use super::kmer_sort::sort_unique_kmers;
pub(super) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
let file = fs::File::open(unique_path)?;
let mmap = unsafe { Mmap::map(&file)? };
@@ -24,7 +21,7 @@ fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
// Sequential constructor: the outer par_iter over partitions already saturates
// the Rayon pool. new_from_par_iter would get no additional threads and adds
// coordination overhead. try_new accesses the same mmap'd pages at zero extra cost.
Mphf::try_new(kmers, PtrHashParams::<CubicEps>::default_balanced())
Mphf::try_new(kmers, mphf_params())
.ok_or_else(|| io::Error::other("ptr_hash construction failed"))
}
@@ -25,17 +25,13 @@ use std::io;
use std::path::Path;
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::prelude::*;
use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{KmerLayer, TypedLayer};
use obikindex::layer::{KmerLayer, Mphf, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xxh3Int};
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
pub(crate) trait PrivateBuilder {
/// Mark scatter as complete (`IndexState::Scattered`).