Implement MPHF migration tool and sparse matrix permutation logic
Refactors sparse matrix handling to implement row permutation logic and introduces a standalone tool for migrating indexes from MPHF 1.x to 2.x formats.
This commit is contained in:
1 parent
d686d39506
commit
8b87078478
24 files changed
+3982
-19
No files matched your search
@@ -0,0 +1,90 @@
|
||||
//! Check the staged files against the layer as it still is, before anything
|
||||
//! is replaced.
|
||||
//!
|
||||
//! For a sample of k-mers read from `unitigs.bin`, the old chain (old MPHF →
|
||||
//! old slot → old data) must agree with the new one (staged MPHF, loaded the
|
||||
//! way queries load it → new slot → staged data): same evidence, same
|
||||
//! fingerprint, same presence set, same counts. A full check is the same loop
|
||||
//! with every k-mer.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Result, anyhow, ensure};
|
||||
use epserde::prelude::*;
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseCompactIntMatrix};
|
||||
use obikindex::layer::Mphf;
|
||||
use obikindex::layer::evidence::Evidence;
|
||||
use obikindex::layer::fingerprint::FingerprintVec;
|
||||
use obiskio::CanonicalKmerIter;
|
||||
|
||||
use crate::old::OldMphf;
|
||||
use crate::plan::Plan;
|
||||
use crate::util::{oki, sk};
|
||||
|
||||
/// Both sides of one per-slot file, when the layer has it.
|
||||
struct Pair<O, N> {
|
||||
old: O,
|
||||
new: N,
|
||||
}
|
||||
|
||||
fn pair<O, N>(
|
||||
exists: bool,
|
||||
old: impl FnOnce() -> Result<O>,
|
||||
new: impl FnOnce() -> Result<N>,
|
||||
) -> Result<Option<Pair<O, N>>> {
|
||||
Ok(if exists { Some(Pair { old: old()?, new: new()? }) } else { None })
|
||||
}
|
||||
|
||||
/// Returns how many k-mers were compared.
|
||||
pub fn check(dir: &Path, staging: &Path, plan: &Plan, old_mphf: &OldMphf, stride: usize) -> Result<usize> {
|
||||
let staged_mphf = unsafe { Mphf::mmap(staging.join("mphf.bin"), Flags::empty()) }
|
||||
.map_err(|e| anyhow!("staged mphf.bin does not load: {e}"))?;
|
||||
let new_mphf = staged_mphf.uncase();
|
||||
|
||||
let evidence = pair(
|
||||
staging.join("evidence.bin").exists(),
|
||||
|| oki(Evidence::open(&dir.join("evidence.bin"))),
|
||||
|| oki(Evidence::open(&staging.join("evidence.bin"))),
|
||||
)?;
|
||||
let fingerprint = pair(
|
||||
staging.join("fingerprint.bin").exists(),
|
||||
|| oki(FingerprintVec::open(&dir.join("fingerprint.bin"))),
|
||||
|| oki(FingerprintVec::open(&staging.join("fingerprint.bin"))),
|
||||
)?;
|
||||
let presence = pair(
|
||||
staging.join("presence").is_dir(),
|
||||
|| Ok(PersistentBitMatrix::open(dir)?),
|
||||
|| Ok(PersistentSparseBitMatrix::open(&staging.join("presence"))?),
|
||||
)?;
|
||||
let counts = pair(
|
||||
staging.join("counts").is_dir(),
|
||||
|| Ok(PersistentIntMatrix::open(dir)?),
|
||||
|| Ok(PersistentSparseCompactIntMatrix::open(&staging.join("counts"))?),
|
||||
)?;
|
||||
|
||||
let mut compared = 0;
|
||||
for (i, kmer) in sk(CanonicalKmerIter::new(&dir.join("unitigs.bin")))?.enumerate() {
|
||||
if i % stride != 0 {
|
||||
continue;
|
||||
}
|
||||
let key = kmer.raw();
|
||||
let o = old_mphf.index(&key);
|
||||
let s = new_mphf.index(&key);
|
||||
ensure!(s < plan.n && plan.new_to_old[s] as usize == o, "k-mer #{i}: staged MPHF slot {s} disagrees with the plan (old slot {o})");
|
||||
|
||||
if let Some(p) = &evidence {
|
||||
ensure!(p.old.decode(o) == p.new.decode(s), "k-mer #{i}: evidence differs (old slot {o}, new slot {s})");
|
||||
}
|
||||
if let Some(p) = &fingerprint {
|
||||
ensure!(p.old.get(o) == p.new.get(s), "k-mer #{i}: fingerprint differs (old slot {o}, new slot {s})");
|
||||
}
|
||||
if let Some(p) = &presence {
|
||||
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: presence row differs (old slot {o}, new slot {s})");
|
||||
}
|
||||
if let Some(p) = &counts {
|
||||
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: count row differs (old slot {o}, new slot {s})");
|
||||
}
|
||||
compared += 1;
|
||||
}
|
||||
Ok(compared)
|
||||
}
|
||||
Reference in new issue
Block a user