Implement MPHF migration tool and sparse matrix permutation logic
Refactors sparse matrix handling to implement row permutation logic and introduces a standalone tool for migrating indexes from MPHF 1.x to 2.x formats.
This commit is contained in:
1 parent
d686d39506
commit
8b87078478
24 files changed
+3982
-19
No files matched your search
Generated
-2
@@ -1528,7 +1528,6 @@ dependencies = [
|
||||
name = "obikindexer"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"cacheline-ef",
|
||||
"epserde",
|
||||
"memmap2",
|
||||
"niffler",
|
||||
@@ -1543,7 +1542,6 @@ dependencies = [
|
||||
"obiskbuilder",
|
||||
"obiskio",
|
||||
"obisys",
|
||||
"ptr_hash",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
||||
@@ -514,6 +514,31 @@ impl PersistentSparseBitMatrixBuilder {
|
||||
}
|
||||
Ok(builder)
|
||||
}
|
||||
|
||||
/// Builds the row-permuted copy of `src` into `dir`: row `s` of the
|
||||
/// result is row `new_to_old[s]` of `src`. `new_to_old` must have
|
||||
/// `src.n()` entries and be a permutation of `0..src.n()` — the caller
|
||||
/// proves that, this only reads `src` at those slots.
|
||||
///
|
||||
/// Needed whenever the slot numbering of a layer changes (its MPHF is
|
||||
/// rebuilt) while the data attached to each k-mer must follow it.
|
||||
/// Rows are read at arbitrary slots of `src`, so convert a dense source
|
||||
/// to sparse first (`build_from_dense`) rather than permuting it
|
||||
/// cell by cell.
|
||||
pub fn build_permuted(
|
||||
src: &PersistentSparseBitMatrix,
|
||||
new_to_old: &[u32],
|
||||
dir: &Path,
|
||||
) -> io::Result<Self> {
|
||||
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
|
||||
let mut genomes: Vec<u32> = Vec::new();
|
||||
for &old in new_to_old {
|
||||
genomes.clear();
|
||||
src.for_each_genome_in_row(old as usize, |g| genomes.push(g as u32));
|
||||
builder.push_row(&genomes);
|
||||
}
|
||||
Ok(builder)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Trait impls ───────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -548,6 +548,29 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
||||
Ok(builder)
|
||||
}
|
||||
|
||||
/// Row-permuted copy of `src`: row `s` of the result is row
|
||||
/// `new_to_old[s]` of `src`. Same contract as
|
||||
/// [`PersistentSparseBitMatrixBuilder::build_permuted`].
|
||||
pub fn build_permuted(
|
||||
src: &PersistentSparseCompactIntMatrix,
|
||||
new_to_old: &[u32],
|
||||
dir: &Path,
|
||||
) -> io::Result<Self> {
|
||||
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
|
||||
let mut cols: Vec<u32> = Vec::new();
|
||||
let mut vals: Vec<u32> = Vec::new();
|
||||
for &old in new_to_old {
|
||||
cols.clear();
|
||||
vals.clear();
|
||||
src.for_each_cell_in_row(old as usize, |c, v| {
|
||||
cols.push(c as u32);
|
||||
vals.push(v);
|
||||
});
|
||||
builder.push_row(&cols, &vals);
|
||||
}
|
||||
Ok(builder)
|
||||
}
|
||||
|
||||
pub fn close(self) -> io::Result<()> {
|
||||
let Self {
|
||||
support,
|
||||
|
||||
@@ -415,3 +415,31 @@ fn partial_jaccard_and_hamming_match_dense_including_diagonal() {
|
||||
drop(dir_dense);
|
||||
drop(dir_sparse);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_permuted_follows_the_permutation() {
|
||||
// Mix of singleton and multi rows, including a repeated multi set, so the
|
||||
// dictionary dedup is exercised on the permuted order too.
|
||||
let rows: Vec<&[bool]> = vec![
|
||||
&[true, false, false, false],
|
||||
&[false, false, true, false],
|
||||
&[true, true, false, false],
|
||||
&[false, false, true, true],
|
||||
&[true, true, false, false],
|
||||
];
|
||||
let (_dir, src) = make_sparse(&rows, 4);
|
||||
let new_to_old: Vec<u32> = vec![3, 0, 4, 1, 2];
|
||||
|
||||
let out = tempdir().unwrap();
|
||||
let permuted =
|
||||
PersistentSparseBitMatrixBuilder::build_permuted(&src, &new_to_old, &out.path().join("p"))
|
||||
.unwrap()
|
||||
.finish()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(permuted.n(), src.n());
|
||||
assert_eq!(permuted.n_cols(), src.n_cols());
|
||||
for (new, &old) in new_to_old.iter().enumerate() {
|
||||
assert_eq!(row_as_bool(&permuted, new), row_as_bool(&src, old as usize), "new row {new}");
|
||||
}
|
||||
}
|
||||
@@ -304,3 +304,32 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
|
||||
assert_eq!(&*sparse.row(slot), &**expected, "slot {slot}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_permuted_follows_the_permutation() {
|
||||
// Rows sharing a support with different values, plus singletons.
|
||||
let rows: Vec<&[u32]> = vec![
|
||||
&[7, 0, 0, 0],
|
||||
&[0, 0, 3, 0],
|
||||
&[2, 5, 0, 0],
|
||||
&[9, 1, 0, 0],
|
||||
&[0, 0, 4, 6],
|
||||
];
|
||||
let (_dir, src) = make_sparse(&rows, 4);
|
||||
let new_to_old: Vec<u32> = vec![4, 2, 0, 3, 1];
|
||||
|
||||
let out = tempdir().unwrap();
|
||||
let permuted = PersistentSparseCompactIntMatrixBuilder::build_permuted(
|
||||
&src,
|
||||
&new_to_old,
|
||||
&out.path().join("p"),
|
||||
)
|
||||
.unwrap()
|
||||
.finish()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(permuted.n(), src.n());
|
||||
for (new, &old) in new_to_old.iter().enumerate() {
|
||||
assert_eq!(permuted.row(new), src.row(old as usize), "new row {new}");
|
||||
}
|
||||
}
|
||||
@@ -210,6 +210,14 @@ impl KmerIndex {
|
||||
crate::partition::index_dir(&self.root_path, i)
|
||||
}
|
||||
|
||||
/// Path of layer `j` of partition `i` — pure path computation, nothing
|
||||
/// is opened (unlike [`layer`](Self::layer)/`layer_dir`, which open the
|
||||
/// layer's MPHF). For maintenance tools that must reach a layer's files
|
||||
/// without going through a format they can't read yet.
|
||||
pub fn layer_path(&self, i: usize, j: usize) -> PathBuf {
|
||||
crate::layer::utils::layer_dir(&self.index_dir(i), j)
|
||||
}
|
||||
|
||||
/// Number of layers in partition `i` — a disk scan, not a metadata read
|
||||
/// (see [`KmerPartition::n_layers`](crate::partition::KmerPartition::n_layers)).
|
||||
pub fn n_layers(&self, i: usize) -> OKIResult<usize> {
|
||||
|
||||
@@ -9,5 +9,7 @@ pub(crate) mod utils;
|
||||
pub use crate::index::error::{OKIError, OKIResult};
|
||||
pub use content_layer::KmerLayer;
|
||||
pub use meta::IndexMode;
|
||||
pub use mphf_layer::{EvidenceKind, KmerBatchIter, KmerIter, MphfLayer, MphfOnly, UnitigIter};
|
||||
pub use mphf_layer::{
|
||||
EvidenceKind, KmerBatchIter, KmerIter, Mphf, MphfLayer, MphfOnly, UnitigIter, mphf_params,
|
||||
};
|
||||
pub use typed_layer::{HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer};
|
||||
@@ -23,7 +23,13 @@ pub(crate) const FINGERPRINT_FILE: &str = "fingerprint.bin";
|
||||
/// The layer MPHF: `Xxh3Int` hasher, `CubicEps` buckets, multi-part (`SINGLE_PART = false`),
|
||||
/// minimal (`REMAP = true`). Built owned (construction + store); queried through
|
||||
/// `MemCase<Mphf>::uncase()`, the ε-copy view into the memory-mapped file.
|
||||
pub(crate) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||
pub type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||
|
||||
/// Construction parameters of [`Mphf`] — the one place they are chosen
|
||||
/// (`lambda = 3.5`, `alpha = 0.99`, `CubicEps`, ~2.4 bits/key).
|
||||
pub fn mphf_params() -> PtrHashParams<CubicEps> {
|
||||
PtrHashParams::<CubicEps>::default_balanced()
|
||||
}
|
||||
|
||||
// ── LayerEvidence ─────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -550,7 +556,7 @@ impl MphfLayer {
|
||||
|
||||
// ── Empty layer ───────────────────────────────────────────────────────
|
||||
if n == 0 {
|
||||
let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::<CubicEps>::default_balanced())
|
||||
let mphf: Mphf = Mphf::try_new(&[] as &[u64], mphf_params())
|
||||
.ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?;
|
||||
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
|
||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||
@@ -574,7 +580,7 @@ impl MphfLayer {
|
||||
let mphf: Mphf = Mphf::new_from_par_iter(
|
||||
n,
|
||||
keys.map(|k| k.raw()).par_bridge(),
|
||||
PtrHashParams::<CubicEps>::default_balanced(),
|
||||
mphf_params(),
|
||||
);
|
||||
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
|
||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||
|
||||
@@ -14,9 +14,7 @@ obicompactvec = { path = "../obicompactvec" }
|
||||
obipipeline = { path = "../obipipeline" }
|
||||
obiread = { path = "../obiread" }
|
||||
obiskbuilder = { path = "../obiskbuilder" }
|
||||
cacheline-ef = "2.0"
|
||||
epserde = "0.13"
|
||||
ptr_hash = { version = "2.1", features = ["epserde"] }
|
||||
niffler = "3.0.0"
|
||||
memmap2 = "0.9.10"
|
||||
rayon = "1"
|
||||
|
||||
@@ -3,18 +3,15 @@ use std::fs;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use cacheline_ef::{CachelineEf, CachelineEfVec};
|
||||
use epserde::ser::Serialize as EpSerialize;
|
||||
use memmap2::Mmap;
|
||||
use obicompactvec::PersistentCompactIntVecBuilder;
|
||||
use obikindex::layer::{Mphf, mphf_params};
|
||||
use obiskio::{SKFileReader, SKResult};
|
||||
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int};
|
||||
use tracing::debug;
|
||||
|
||||
use super::kmer_sort::sort_unique_kmers;
|
||||
|
||||
pub(super) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||
|
||||
fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
|
||||
let file = fs::File::open(unique_path)?;
|
||||
let mmap = unsafe { Mmap::map(&file)? };
|
||||
@@ -24,7 +21,7 @@ fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
|
||||
// Sequential constructor: the outer par_iter over partitions already saturates
|
||||
// the Rayon pool. new_from_par_iter would get no additional threads and adds
|
||||
// coordination overhead. try_new accesses the same mmap'd pages at zero extra cost.
|
||||
Mphf::try_new(kmers, PtrHashParams::<CubicEps>::default_balanced())
|
||||
Mphf::try_new(kmers, mphf_params())
|
||||
.ok_or_else(|| io::Error::other("ptr_hash construction failed"))
|
||||
}
|
||||
|
||||
|
||||
@@ -25,17 +25,13 @@ use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
|
||||
use cacheline_ef::{CachelineEf, CachelineEfVec};
|
||||
use epserde::prelude::*;
|
||||
use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec};
|
||||
use obidebruinj::GraphDeBruijn;
|
||||
use obikindex::layer::IndexMode;
|
||||
use obikindex::layer::{KmerLayer, TypedLayer};
|
||||
use obikindex::layer::{KmerLayer, Mphf, TypedLayer};
|
||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
||||
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xxh3Int};
|
||||
|
||||
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||
|
||||
pub(crate) trait PrivateBuilder {
|
||||
/// Mark scatter as complete (`IndexState::Scattered`).
|
||||
|
||||
Reference in new issue
Block a user