Refactors sparse matrix handling to implement row permutation logic and introduces a standalone tool for migrating indexes from MPHF 1.x to 2.x formats.
91 lines
3.4 KiB
Rust
91 lines
3.4 KiB
Rust
//! Check the staged files against the layer as it still is, before anything
|
|
//! is replaced.
|
|
//!
|
|
//! For a sample of k-mers read from `unitigs.bin`, the old chain (old MPHF →
|
|
//! old slot → old data) must agree with the new one (staged MPHF, loaded the
|
|
//! way queries load it → new slot → staged data): same evidence, same
|
|
//! fingerprint, same presence set, same counts. A full check is the same loop
|
|
//! with every k-mer.
|
|
|
|
use std::path::Path;
|
|
|
|
use anyhow::{Result, anyhow, ensure};
|
|
use epserde::prelude::*;
|
|
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseCompactIntMatrix};
|
|
use obikindex::layer::Mphf;
|
|
use obikindex::layer::evidence::Evidence;
|
|
use obikindex::layer::fingerprint::FingerprintVec;
|
|
use obiskio::CanonicalKmerIter;
|
|
|
|
use crate::old::OldMphf;
|
|
use crate::plan::Plan;
|
|
use crate::util::{oki, sk};
|
|
|
|
/// Both sides of one per-slot file, when the layer has it.
|
|
struct Pair<O, N> {
|
|
old: O,
|
|
new: N,
|
|
}
|
|
|
|
fn pair<O, N>(
|
|
exists: bool,
|
|
old: impl FnOnce() -> Result<O>,
|
|
new: impl FnOnce() -> Result<N>,
|
|
) -> Result<Option<Pair<O, N>>> {
|
|
Ok(if exists { Some(Pair { old: old()?, new: new()? }) } else { None })
|
|
}
|
|
|
|
/// Returns how many k-mers were compared.
|
|
pub fn check(dir: &Path, staging: &Path, plan: &Plan, old_mphf: &OldMphf, stride: usize) -> Result<usize> {
|
|
let staged_mphf = unsafe { Mphf::mmap(staging.join("mphf.bin"), Flags::empty()) }
|
|
.map_err(|e| anyhow!("staged mphf.bin does not load: {e}"))?;
|
|
let new_mphf = staged_mphf.uncase();
|
|
|
|
let evidence = pair(
|
|
staging.join("evidence.bin").exists(),
|
|
|| oki(Evidence::open(&dir.join("evidence.bin"))),
|
|
|| oki(Evidence::open(&staging.join("evidence.bin"))),
|
|
)?;
|
|
let fingerprint = pair(
|
|
staging.join("fingerprint.bin").exists(),
|
|
|| oki(FingerprintVec::open(&dir.join("fingerprint.bin"))),
|
|
|| oki(FingerprintVec::open(&staging.join("fingerprint.bin"))),
|
|
)?;
|
|
let presence = pair(
|
|
staging.join("presence").is_dir(),
|
|
|| Ok(PersistentBitMatrix::open(dir)?),
|
|
|| Ok(PersistentSparseBitMatrix::open(&staging.join("presence"))?),
|
|
)?;
|
|
let counts = pair(
|
|
staging.join("counts").is_dir(),
|
|
|| Ok(PersistentIntMatrix::open(dir)?),
|
|
|| Ok(PersistentSparseCompactIntMatrix::open(&staging.join("counts"))?),
|
|
)?;
|
|
|
|
let mut compared = 0;
|
|
for (i, kmer) in sk(CanonicalKmerIter::new(&dir.join("unitigs.bin")))?.enumerate() {
|
|
if i % stride != 0 {
|
|
continue;
|
|
}
|
|
let key = kmer.raw();
|
|
let o = old_mphf.index(&key);
|
|
let s = new_mphf.index(&key);
|
|
ensure!(s < plan.n && plan.new_to_old[s] as usize == o, "k-mer #{i}: staged MPHF slot {s} disagrees with the plan (old slot {o})");
|
|
|
|
if let Some(p) = &evidence {
|
|
ensure!(p.old.decode(o) == p.new.decode(s), "k-mer #{i}: evidence differs (old slot {o}, new slot {s})");
|
|
}
|
|
if let Some(p) = &fingerprint {
|
|
ensure!(p.old.get(o) == p.new.get(s), "k-mer #{i}: fingerprint differs (old slot {o}, new slot {s})");
|
|
}
|
|
if let Some(p) = &presence {
|
|
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: presence row differs (old slot {o}, new slot {s})");
|
|
}
|
|
if let Some(p) = &counts {
|
|
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: count row differs (old slot {o}, new slot {s})");
|
|
}
|
|
compared += 1;
|
|
}
|
|
Ok(compared)
|
|
}
|