Files
obikmer/tools/migrate-mphf/src/verify.rs
T
Eric Coissac 8b87078478 Implement MPHF migration tool and sparse matrix permutation logic
Refactors sparse matrix handling to implement row permutation logic and introduces a standalone tool for migrating indexes from MPHF 1.x to 2.x formats.
2026-10-15 12:12:51 +02:00

91 lines
3.4 KiB
Rust

//! Check the staged files against the layer as it still is, before anything
//! is replaced.
//!
//! For a sample of k-mers read from `unitigs.bin`, the old chain (old MPHF →
//! old slot → old data) must agree with the new one (staged MPHF, loaded the
//! way queries load it → new slot → staged data): same evidence, same
//! fingerprint, same presence set, same counts. A full check is the same loop
//! with every k-mer.
use std::path::Path;
use anyhow::{Result, anyhow, ensure};
use epserde::prelude::*;
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseCompactIntMatrix};
use obikindex::layer::Mphf;
use obikindex::layer::evidence::Evidence;
use obikindex::layer::fingerprint::FingerprintVec;
use obiskio::CanonicalKmerIter;
use crate::old::OldMphf;
use crate::plan::Plan;
use crate::util::{oki, sk};
/// Both sides of one per-slot file, when the layer has it.
struct Pair<O, N> {
old: O,
new: N,
}
fn pair<O, N>(
exists: bool,
old: impl FnOnce() -> Result<O>,
new: impl FnOnce() -> Result<N>,
) -> Result<Option<Pair<O, N>>> {
Ok(if exists { Some(Pair { old: old()?, new: new()? }) } else { None })
}
/// Returns how many k-mers were compared.
pub fn check(dir: &Path, staging: &Path, plan: &Plan, old_mphf: &OldMphf, stride: usize) -> Result<usize> {
let staged_mphf = unsafe { Mphf::mmap(staging.join("mphf.bin"), Flags::empty()) }
.map_err(|e| anyhow!("staged mphf.bin does not load: {e}"))?;
let new_mphf = staged_mphf.uncase();
let evidence = pair(
staging.join("evidence.bin").exists(),
|| oki(Evidence::open(&dir.join("evidence.bin"))),
|| oki(Evidence::open(&staging.join("evidence.bin"))),
)?;
let fingerprint = pair(
staging.join("fingerprint.bin").exists(),
|| oki(FingerprintVec::open(&dir.join("fingerprint.bin"))),
|| oki(FingerprintVec::open(&staging.join("fingerprint.bin"))),
)?;
let presence = pair(
staging.join("presence").is_dir(),
|| Ok(PersistentBitMatrix::open(dir)?),
|| Ok(PersistentSparseBitMatrix::open(&staging.join("presence"))?),
)?;
let counts = pair(
staging.join("counts").is_dir(),
|| Ok(PersistentIntMatrix::open(dir)?),
|| Ok(PersistentSparseCompactIntMatrix::open(&staging.join("counts"))?),
)?;
let mut compared = 0;
for (i, kmer) in sk(CanonicalKmerIter::new(&dir.join("unitigs.bin")))?.enumerate() {
if i % stride != 0 {
continue;
}
let key = kmer.raw();
let o = old_mphf.index(&key);
let s = new_mphf.index(&key);
ensure!(s < plan.n && plan.new_to_old[s] as usize == o, "k-mer #{i}: staged MPHF slot {s} disagrees with the plan (old slot {o})");
if let Some(p) = &evidence {
ensure!(p.old.decode(o) == p.new.decode(s), "k-mer #{i}: evidence differs (old slot {o}, new slot {s})");
}
if let Some(p) = &fingerprint {
ensure!(p.old.get(o) == p.new.get(s), "k-mer #{i}: fingerprint differs (old slot {o}, new slot {s})");
}
if let Some(p) = &presence {
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: presence row differs (old slot {o}, new slot {s})");
}
if let Some(p) = &counts {
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: count row differs (old slot {o}, new slot {s})");
}
compared += 1;
}
Ok(compared)
}