Implement MPHF migration tool and sparse matrix permutation logic

Refactors sparse matrix handling to implement row permutation logic and introduces a standalone tool for migrating indexes from MPHF 1.x to 2.x formats.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:12:51 +02:00
1 parent d686d39506
commit 8b87078478
24 files changed
+3982 -19

No files matched your search

+3216
View File
File diff suppressed because it is too large. Load diff
+36
View File
@@ -0,0 +1,36 @@
[package]
name = "migrate-mphf"
version = "0.1.0"
edition = "2024"
publish = false
description = "One-shot converter: obikmer indexes whose layer MPHFs were built with ptr_hash 1.x -> ptr_hash 2.x"
# Standalone: not a member of src/'s workspace, own Cargo.lock. The previous
# generation of the MPHF stack (and its `proc-macro-error2` future-incompat
# warning) lives only here, never in obikmer itself.
[workspace]
[features]
default = []
numa = ["obisys/numa"]
[dependencies]
obikindex = { path = "../../src/obikindex" }
obicompactvec = { path = "../../src/obicompactvec" }
obiskio = { path = "../../src/obiskio" }
obisys = { path = "../../src/obisys", default-features = false }
epserde = "0.13"
rayon = "1"
anyhow = "1"
clap = { version = "4", features = ["derive"] }
tracing = "0.1.44"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
# The generation the indexes to convert were built with, under other names
# so that both can live in this one binary.
ptr_hash_old = { package = "ptr_hash", version = "=1.1.0" }
epserde_old = { package = "epserde", version = "=0.8.0" }
cacheline_ef_old = { package = "cacheline-ef", version = "=1.1.0" }
[profile.release]
debug = 1
+98
View File
@@ -0,0 +1,98 @@
//! Crash-safe replacement of a layer's files by their converted versions.
//!
//! Everything new is written under `<layer>/.migrate_mphf/` first; nothing of
//! the layer is touched until that is complete and verified. Then a `COMMIT`
//! marker is written, and [`finish`] moves the staged files into place. Every
//! step of [`finish`] is idempotent ("if the staged item still exists, move
//! it"), so a layer interrupted after `COMMIT` is simply finished by the next
//! run. Files are only ever replaced by `rename`, never written in place:
//! some of them are hard-linked from elsewhere (`ls -l` link count 2), and an
//! in-place write would corrupt the other link.
use std::fs;
use std::io;
use std::path::Path;
use anyhow::{Context, Result};
pub const STAGING: &str = ".migrate_mphf";
pub const COMMIT: &str = "COMMIT";
/// Written last: the layer is converted, skip it on later runs.
pub const DONE: &str = ".mphf_migrated";
/// Where derived annexes go instead of being deleted.
pub const STALE_DIR: &str = "stale_annexes_pre_ptrhash2";
/// Replaced as directories.
const DIRS: [&str; 2] = ["presence", "counts"];
/// Replaced as files; `mphf.bin` last.
const FILES: [&str; 3] = ["evidence.bin", "fingerprint.bin", "mphf.bin"];
/// Slot-ordered or slot-derived annexes of a layer: wrong (silently) once the
/// slot numbering changes, so they are set aside, to be rebuilt.
const LAYER_ANNEXES: [&str; 7] = [
"siblings.psib",
"entropy.pent",
"entropy.noamb.pent",
"pairweight.ppwt",
"pairweight.noamb.ppwt",
"quartet.pbiv",
"quartet.noamb.pbiv",
];
/// Index-level annexes derived from the sibling annex.
pub const INDEX_ANNEXES: [&str; 2] = ["family_overlap.povl", "family_overlap.noamb.povl"];
/// Move the staged files into `dir`, set the stale annexes aside, mark the
/// layer done and drop the staging area.
pub fn finish(dir: &Path) -> Result<()> {
let staging = dir.join(STAGING);
for name in DIRS {
let staged = staging.join(name);
if staged.exists() {
let target = dir.join(name);
if target.exists() {
fs::remove_dir_all(&target).with_context(|| format!("removing {}", target.display()))?;
}
fs::rename(&staged, &target).with_context(|| format!("installing {}", target.display()))?;
}
}
for name in FILES {
let staged = staging.join(name);
if staged.exists() {
let target = dir.join(name);
fs::rename(&staged, &target).with_context(|| format!("installing {}", target.display()))?;
}
}
set_aside(dir, &LAYER_ANNEXES)?;
fs::write(dir.join(DONE), env!("CARGO_PKG_VERSION"))?;
fs::remove_dir_all(&staging)?;
Ok(())
}
/// Move each of `names` that exists in `dir` into `dir/STALE_DIR/`.
/// Returns how many were moved.
pub fn set_aside(dir: &Path, names: &[&str]) -> Result<usize> {
let aside = dir.join(STALE_DIR);
let mut moved = 0;
for name in names {
let from = dir.join(name);
if !from.exists() {
continue;
}
fs::create_dir_all(&aside)?;
fs::rename(&from, aside.join(name)).with_context(|| format!("setting aside {}", from.display()))?;
moved += 1;
}
Ok(moved)
}
/// Remove a staging area left by an interrupted, uncommitted run.
pub fn discard(dir: &Path) -> io::Result<()> {
match fs::remove_dir_all(dir.join(STAGING)) {
Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e),
_ => Ok(()),
}
}
+100
View File
@@ -0,0 +1,100 @@
//! Permute the per-slot files of a layer into `staging/`.
//!
//! Presence and count matrices always come out in the compact row-major
//! (sparse) format, whatever they were: a dense source is first converted
//! sequentially (`build_from_dense`), then permuted from the sparse form,
//! whose rows are cheap to read at arbitrary slots.
use std::fs;
use std::path::Path;
use anyhow::{Result, ensure};
use obicompactvec::{
PersistentBitMatrix, PersistentIntMatrix, PersistentSparseBitMatrixBuilder,
PersistentSparseCompactIntMatrixBuilder,
};
use obikindex::layer::evidence::{Evidence, EvidenceWriter};
use obikindex::layer::fingerprint::{FingerprintVec, FingerprintVecWriter};
use crate::plan::Plan;
use crate::util::oki;
/// Every per-slot file of the layer in `dir` that exists, permuted into `staging`.
pub fn permute_all(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
if dir.join("evidence.bin").exists() {
evidence(dir, staging, plan)?;
}
if dir.join("fingerprint.bin").exists() {
fingerprint(dir, staging, plan)?;
}
if dir.join("presence").is_dir() {
presence(dir, staging, plan)?;
}
if dir.join("counts").is_dir() {
counts(dir, staging, plan)?;
}
Ok(())
}
fn evidence(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
let old = oki(Evidence::open(&dir.join("evidence.bin")))?;
ensure!(old.len() == plan.n, "evidence.bin has {} slots, layer has {}", old.len(), plan.n);
let mut out = EvidenceWriter::new(plan.n);
for (s, &o) in plan.new_to_old.iter().enumerate() {
let (chunk_id, rank) = old.decode(o as usize);
out.set(s, chunk_id, rank);
}
oki(out.write(&staging.join("evidence.bin")))
}
fn fingerprint(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
let old = oki(FingerprintVec::open(&dir.join("fingerprint.bin")))?;
ensure!(old.n() == plan.n, "fingerprint.bin has {} slots, layer has {}", old.n(), plan.n);
let mut out = FingerprintVecWriter::new(plan.n, old.b());
for (s, &o) in plan.new_to_old.iter().enumerate() {
out.set(s, old.get(o as usize));
}
oki(out.write(&staging.join("fingerprint.bin")))
}
fn presence(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
let out = staging.join("presence");
let src = match PersistentBitMatrix::open(dir)? {
// Mono-genome: no file, nothing is attached to slots.
PersistentBitMatrix::Implicit { .. } => return Ok(()),
PersistentBitMatrix::Sparse(s) => {
ensure!(s.n() == plan.n, "presence has {} rows, layer has {}", s.n(), plan.n);
PersistentSparseBitMatrixBuilder::build_permuted(&s, &plan.new_to_old, &out)?.close()?;
return Ok(());
}
dense => dense,
};
ensure!(src.n() == plan.n, "presence has {} rows, layer has {}", src.n(), plan.n);
let tmp = staging.join("presence_dense_to_sparse");
let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&src, &tmp)?.finish()?;
drop(src);
PersistentSparseBitMatrixBuilder::build_permuted(&sparse, &plan.new_to_old, &out)?.close()?;
drop(sparse);
fs::remove_dir_all(&tmp)?;
Ok(())
}
fn counts(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
let out = staging.join("counts");
let src = match PersistentIntMatrix::open(dir)? {
PersistentIntMatrix::Sparse(s) => {
ensure!(s.n() == plan.n, "counts has {} rows, layer has {}", s.n(), plan.n);
PersistentSparseCompactIntMatrixBuilder::build_permuted(&s, &plan.new_to_old, &out)?.close()?;
return Ok(());
}
dense => dense,
};
ensure!(src.n() == plan.n, "counts has {} rows, layer has {}", src.n(), plan.n);
let tmp = staging.join("counts_dense_to_sparse");
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&src, &tmp)?.finish()?;
drop(src);
PersistentSparseCompactIntMatrixBuilder::build_permuted(&sparse, &plan.new_to_old, &out)?.close()?;
drop(sparse);
fs::remove_dir_all(&tmp)?;
Ok(())
}
+75
View File
@@ -0,0 +1,75 @@
//! Convert one layer directory: new MPHF, permuted per-slot data, verified,
//! then installed atomically (see `commit`).
use std::fs;
use std::path::Path;
use anyhow::{Result, anyhow, ensure};
use epserde::prelude::*;
use obikindex::layer::{Mphf, mphf_params};
use obiskio::UnitigFileReader;
use crate::util::sk;
use crate::{commit, data, old, plan, verify};
pub struct Options {
/// How many k-mers per layer to compare old against new (ignored with `verify_full`).
pub verify_sample: usize,
pub verify_full: bool,
/// Do everything except installing the result.
pub dry_run: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Outcome {
/// Already converted by an earlier run.
Skipped,
/// Interrupted after verification by an earlier run; installed now.
Resumed,
Converted { n: usize, verified: usize },
/// Everything but the installation.
Checked { n: usize, verified: usize },
}
pub fn convert(dir: &Path, opts: &Options) -> Result<Outcome> {
if dir.join(commit::DONE).exists() {
commit::discard(dir)?;
return Ok(Outcome::Skipped);
}
let staging = dir.join(commit::STAGING);
if staging.join(commit::COMMIT).exists() {
commit::finish(dir)?;
return Ok(Outcome::Resumed);
}
commit::discard(dir)?;
fs::create_dir_all(&staging)?;
let n = sk(UnitigFileReader::open_sequential(&dir.join("unitigs.bin")))?.n_kmers();
ensure!(n <= u32::MAX as usize, "layer has {n} k-mers, more than the 2^32 this tool handles");
let verified = if n == 0 {
empty_mphf(&staging)?;
0
} else {
let old_mphf = old::load(&dir.join("mphf.bin"))?;
let plan = plan::build(dir, &staging, n, &old_mphf)?;
data::permute_all(dir, &staging, &plan)?;
let stride = if opts.verify_full { 1 } else { (n / opts.verify_sample.max(1)).max(1) };
verify::check(dir, &staging, &plan, &old_mphf, stride)?
};
if opts.dry_run {
commit::discard(dir)?;
return Ok(Outcome::Checked { n, verified });
}
fs::write(staging.join(commit::COMMIT), b"")?;
commit::finish(dir)?;
Ok(Outcome::Converted { n, verified })
}
/// An empty layer still carries an MPHF (of nothing), like `MphfLayer::build` writes.
fn empty_mphf(staging: &Path) -> Result<()> {
let mphf = Mphf::try_new(&[] as &[u64], mphf_params())
.ok_or_else(|| anyhow!("empty MPHF construction failed"))?;
unsafe { mphf.store(staging.join("mphf.bin")) }.map_err(|e| anyhow!("storing the empty MPHF: {e}"))
}
+121
View File
@@ -0,0 +1,121 @@
//! One-shot converter for obikmer indexes whose layer MPHFs were built with
//! `ptr_hash` 1.x: rebuilds each MPHF with `ptr_hash` 2.x and carries the
//! per-slot data (evidence, fingerprint, presence/count matrices) over to the
//! new slot numbering, in place, layer by layer, verified before installing.
//!
//! Presence and count matrices come out in the compact row-major format.
//! Annexes derived from slot numbering (sibling, entropy, pair-weight,
//! quartet, family-overlap) are set aside under `stale_annexes_pre_ptrhash2/`
//! and must be rebuilt.
mod commit;
mod data;
mod layer;
mod old;
mod plan;
mod util;
mod verify;
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use anyhow::{Result, ensure};
use clap::Parser;
use obikindex::KmerIndex;
use obisys::PartitionRunner;
use tracing::{info, warn};
use layer::{Options, Outcome};
use util::oki;
#[derive(Parser)]
#[command(about = "Convert an obikmer index from ptr_hash 1.x to ptr_hash 2.x MPHFs, in place")]
struct Cli {
/// Index directory (the one holding `index.meta` and `part_*`).
index: PathBuf,
/// Limit the number of CPUs used.
#[arg(long)]
cpu_max: Option<usize>,
/// K-mers compared old-vs-new per layer before installing it.
#[arg(long, default_value_t = 200_000)]
verify_sample: usize,
/// Compare every k-mer of every layer (slow).
#[arg(long)]
verify_full: bool,
/// Do everything except installing: convert into staging, verify, discard.
#[arg(long)]
dry_run: bool,
/// Only these partitions (repeatable). The index-level annexes are then left alone.
#[arg(long)]
partition: Vec<usize>,
/// Only these layers of each partition (repeatable). The index-level annexes are then left alone.
#[arg(long)]
layer: Vec<usize>,
}
fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()))
.init();
let cli = Cli::parse();
if let Some(n) = cli.cpu_max {
obisys::set_cpu_cap(n);
}
// Holds the index's directory lock for the whole run, and sets k.
let index = oki(KmerIndex::open_lock(&cli.index))?;
let n_partitions = index.n_partitions();
let partial = !cli.partition.is_empty() || !cli.layer.is_empty();
let order: Vec<usize> = if cli.partition.is_empty() { (0..n_partitions).collect() } else { cli.partition.clone() };
ensure!(order.iter().all(|&p| p < n_partitions), "index has only {n_partitions} partitions");
let opts = Options { verify_sample: cli.verify_sample, verify_full: cli.verify_full, dry_run: cli.dry_run };
let converted = AtomicUsize::new(0);
let skipped = AtomicUsize::new(0);
let mut done = 0usize;
PartitionRunner::new().run(
&order,
|p| -> Result<Vec<(usize, Outcome)>> {
let n_layers = oki(index.n_layers(p))?;
let mut outcomes = Vec::new();
for l in (0..n_layers).filter(|l| cli.layer.is_empty() || cli.layer.contains(l)) {
let dir = index.layer_path(p, l);
let outcome = layer::convert(&dir, &opts)
.map_err(|e| e.context(format!("partition {p}, layer {l} ({})", dir.display())))?;
match outcome {
Outcome::Skipped => skipped.fetch_add(1, Ordering::Relaxed),
_ => converted.fetch_add(1, Ordering::Relaxed),
};
outcomes.push((l, outcome));
}
Ok(outcomes)
},
|p, outcomes, elapsed| {
done += 1;
info!("partition {p} ({done}/{}) in {:.1?}: {outcomes:?}", order.len(), elapsed);
},
)?;
if !cli.dry_run && !partial {
let moved = commit::set_aside(index.dir(), &commit::INDEX_ANNEXES)?;
if moved > 0 {
info!("{moved} index-level annex(es) set aside under {}", commit::STALE_DIR);
}
} else if !cli.dry_run {
warn!("partial run: index-level annexes (family overlap) not set aside; do it on the final pass");
}
info!(
"done: {} layer(s) {}, {} already converted",
converted.load(Ordering::Relaxed),
if cli.dry_run { "checked" } else { "converted" },
skipped.load(Ordering::Relaxed)
);
Ok(())
}
+17
View File
@@ -0,0 +1,17 @@
//! The previous MPHF generation (`ptr_hash` 1.x), kept only to read the
//! `mphf.bin` files of indexes built before the migration.
use std::path::Path;
use anyhow::{Result, anyhow};
use cacheline_ef_old::{CachelineEf, CachelineEfVec};
use epserde_old::prelude::*;
use ptr_hash_old::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
/// Exactly the type `obikindex` used before the migration.
pub type OldMphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
pub fn load(path: &Path) -> Result<OldMphf> {
OldMphf::load_full(path)
.map_err(|e| anyhow!("{}: not a ptr_hash 1.x MPHF (already converted?): {e}", path.display()))
}
+53
View File
@@ -0,0 +1,53 @@
//! The slot permutation between a layer's old and new MPHF.
//!
//! A layer's MPHF gives each k-mer a slot, and every other per-slot file of
//! the layer (evidence, fingerprint, presence/count matrices) is indexed by
//! it. A new MPHF numbers the same k-mers differently, so the data has to
//! follow: the k-mers themselves come from `unitigs.bin`, which does not
//! depend on the MPHF at all.
use std::path::Path;
use anyhow::{Result, anyhow, ensure};
use epserde::prelude::*;
use obikindex::layer::{Mphf, mphf_params};
use obiskio::CanonicalKmerIter;
use rayon::prelude::*;
use crate::old::OldMphf;
use crate::util::sk;
/// `new_to_old[s]` is the old slot of the k-mer that the new MPHF puts in
/// slot `s`. Always a bijection on `0..n` (checked while it is built).
pub struct Plan {
pub n: usize,
pub new_to_old: Vec<u32>,
}
/// Build the new MPHF of the layer whose `unitigs.bin` is in `dir`, store it
/// as `staging/mphf.bin`, and derive the permutation against `old`.
pub fn build(dir: &Path, staging: &Path, n: usize, old: &OldMphf) -> Result<Plan> {
let unitigs = dir.join("unitigs.bin");
// Same construction as `MphfLayer::build`: a clonable mmap iterator, bridged.
let keys = sk(CanonicalKmerIter::new(&unitigs))?;
let new: Mphf = Mphf::new_from_par_iter(n, keys.map(|k| k.raw()).par_bridge(), mphf_params());
unsafe { new.store(staging.join("mphf.bin")) }.map_err(|e| anyhow!("storing the new MPHF: {e}"))?;
let mut new_to_old = vec![u32::MAX; n];
let mut old_seen = vec![0u8; n.div_ceil(8)];
for kmer in sk(CanonicalKmerIter::new(&unitigs))? {
let key = kmer.raw();
let o = old.index(&key);
let s = new.index(&key);
ensure!(o < n, "old MPHF: slot {o} out of range (n = {n}) — is this layer's mphf.bin really the one of its unitigs.bin?");
ensure!(s < n, "new MPHF: slot {s} out of range (n = {n})");
ensure!(new_to_old[s] == u32::MAX, "new MPHF puts two k-mers in slot {s}");
let (byte, bit) = (o / 8, 1u8 << (o % 8));
ensure!(old_seen[byte] & bit == 0, "old MPHF puts two k-mers in slot {o}");
old_seen[byte] |= bit;
new_to_old[s] = o as u32;
}
// n distinct new slots, all in range, among n: the map is total.
Ok(Plan { n, new_to_old })
}
+11
View File
@@ -0,0 +1,11 @@
//! Error adapters: the libraries each have their own error type.
use anyhow::{Result, anyhow};
pub fn sk<T>(r: obiskio::SKResult<T>) -> Result<T> {
r.map_err(|e| anyhow!("{e}"))
}
pub fn oki<T>(r: obikindex::OKIResult<T>) -> Result<T> {
r.map_err(|e| anyhow!("{e}"))
}
+90
View File
@@ -0,0 +1,90 @@
//! Check the staged files against the layer as it still is, before anything
//! is replaced.
//!
//! For a sample of k-mers read from `unitigs.bin`, the old chain (old MPHF →
//! old slot → old data) must agree with the new one (staged MPHF, loaded the
//! way queries load it → new slot → staged data): same evidence, same
//! fingerprint, same presence set, same counts. A full check is the same loop
//! with every k-mer.
use std::path::Path;
use anyhow::{Result, anyhow, ensure};
use epserde::prelude::*;
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseCompactIntMatrix};
use obikindex::layer::Mphf;
use obikindex::layer::evidence::Evidence;
use obikindex::layer::fingerprint::FingerprintVec;
use obiskio::CanonicalKmerIter;
use crate::old::OldMphf;
use crate::plan::Plan;
use crate::util::{oki, sk};
/// Both sides of one per-slot file, when the layer has it.
struct Pair<O, N> {
old: O,
new: N,
}
fn pair<O, N>(
exists: bool,
old: impl FnOnce() -> Result<O>,
new: impl FnOnce() -> Result<N>,
) -> Result<Option<Pair<O, N>>> {
Ok(if exists { Some(Pair { old: old()?, new: new()? }) } else { None })
}
/// Returns how many k-mers were compared.
pub fn check(dir: &Path, staging: &Path, plan: &Plan, old_mphf: &OldMphf, stride: usize) -> Result<usize> {
let staged_mphf = unsafe { Mphf::mmap(staging.join("mphf.bin"), Flags::empty()) }
.map_err(|e| anyhow!("staged mphf.bin does not load: {e}"))?;
let new_mphf = staged_mphf.uncase();
let evidence = pair(
staging.join("evidence.bin").exists(),
|| oki(Evidence::open(&dir.join("evidence.bin"))),
|| oki(Evidence::open(&staging.join("evidence.bin"))),
)?;
let fingerprint = pair(
staging.join("fingerprint.bin").exists(),
|| oki(FingerprintVec::open(&dir.join("fingerprint.bin"))),
|| oki(FingerprintVec::open(&staging.join("fingerprint.bin"))),
)?;
let presence = pair(
staging.join("presence").is_dir(),
|| Ok(PersistentBitMatrix::open(dir)?),
|| Ok(PersistentSparseBitMatrix::open(&staging.join("presence"))?),
)?;
let counts = pair(
staging.join("counts").is_dir(),
|| Ok(PersistentIntMatrix::open(dir)?),
|| Ok(PersistentSparseCompactIntMatrix::open(&staging.join("counts"))?),
)?;
let mut compared = 0;
for (i, kmer) in sk(CanonicalKmerIter::new(&dir.join("unitigs.bin")))?.enumerate() {
if i % stride != 0 {
continue;
}
let key = kmer.raw();
let o = old_mphf.index(&key);
let s = new_mphf.index(&key);
ensure!(s < plan.n && plan.new_to_old[s] as usize == o, "k-mer #{i}: staged MPHF slot {s} disagrees with the plan (old slot {o})");
if let Some(p) = &evidence {
ensure!(p.old.decode(o) == p.new.decode(s), "k-mer #{i}: evidence differs (old slot {o}, new slot {s})");
}
if let Some(p) = &fingerprint {
ensure!(p.old.get(o) == p.new.get(s), "k-mer #{i}: fingerprint differs (old slot {o}, new slot {s})");
}
if let Some(p) = &presence {
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: presence row differs (old slot {o}, new slot {s})");
}
if let Some(p) = &counts {
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: count row differs (old slot {o}, new slot {s})");
}
compared += 1;
}
Ok(compared)
}