Implement MPHF migration tool and sparse matrix permutation logic
Refactors sparse matrix handling to implement row permutation logic and introduces a standalone tool for migrating indexes from MPHF 1.x to 2.x formats.
This commit is contained in:
1 parent
d686d39506
commit
8b87078478
24 files changed
+3982
-19
No files matched your search
@@ -0,0 +1,75 @@
|
||||
//! Convert one layer directory: new MPHF, permuted per-slot data, verified,
|
||||
//! then installed atomically (see `commit`).
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Result, anyhow, ensure};
|
||||
use epserde::prelude::*;
|
||||
use obikindex::layer::{Mphf, mphf_params};
|
||||
use obiskio::UnitigFileReader;
|
||||
|
||||
use crate::util::sk;
|
||||
use crate::{commit, data, old, plan, verify};
|
||||
|
||||
pub struct Options {
|
||||
/// How many k-mers per layer to compare old against new (ignored with `verify_full`).
|
||||
pub verify_sample: usize,
|
||||
pub verify_full: bool,
|
||||
/// Do everything except installing the result.
|
||||
pub dry_run: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Outcome {
|
||||
/// Already converted by an earlier run.
|
||||
Skipped,
|
||||
/// Interrupted after verification by an earlier run; installed now.
|
||||
Resumed,
|
||||
Converted { n: usize, verified: usize },
|
||||
/// Everything but the installation.
|
||||
Checked { n: usize, verified: usize },
|
||||
}
|
||||
|
||||
pub fn convert(dir: &Path, opts: &Options) -> Result<Outcome> {
|
||||
if dir.join(commit::DONE).exists() {
|
||||
commit::discard(dir)?;
|
||||
return Ok(Outcome::Skipped);
|
||||
}
|
||||
let staging = dir.join(commit::STAGING);
|
||||
if staging.join(commit::COMMIT).exists() {
|
||||
commit::finish(dir)?;
|
||||
return Ok(Outcome::Resumed);
|
||||
}
|
||||
commit::discard(dir)?;
|
||||
fs::create_dir_all(&staging)?;
|
||||
|
||||
let n = sk(UnitigFileReader::open_sequential(&dir.join("unitigs.bin")))?.n_kmers();
|
||||
ensure!(n <= u32::MAX as usize, "layer has {n} k-mers, more than the 2^32 this tool handles");
|
||||
|
||||
let verified = if n == 0 {
|
||||
empty_mphf(&staging)?;
|
||||
0
|
||||
} else {
|
||||
let old_mphf = old::load(&dir.join("mphf.bin"))?;
|
||||
let plan = plan::build(dir, &staging, n, &old_mphf)?;
|
||||
data::permute_all(dir, &staging, &plan)?;
|
||||
let stride = if opts.verify_full { 1 } else { (n / opts.verify_sample.max(1)).max(1) };
|
||||
verify::check(dir, &staging, &plan, &old_mphf, stride)?
|
||||
};
|
||||
|
||||
if opts.dry_run {
|
||||
commit::discard(dir)?;
|
||||
return Ok(Outcome::Checked { n, verified });
|
||||
}
|
||||
fs::write(staging.join(commit::COMMIT), b"")?;
|
||||
commit::finish(dir)?;
|
||||
Ok(Outcome::Converted { n, verified })
|
||||
}
|
||||
|
||||
/// An empty layer still carries an MPHF (of nothing), like `MphfLayer::build` writes.
|
||||
fn empty_mphf(staging: &Path) -> Result<()> {
|
||||
let mphf = Mphf::try_new(&[] as &[u64], mphf_params())
|
||||
.ok_or_else(|| anyhow!("empty MPHF construction failed"))?;
|
||||
unsafe { mphf.store(staging.join("mphf.bin")) }.map_err(|e| anyhow!("storing the empty MPHF: {e}"))
|
||||
}
|
||||
Reference in new issue
Block a user