Push wpvskkyozppn #78

Merged
coissac merged 15 commits from push-wpvskkyozppn into main 2026-10-15 10:38:59 +00:00
24 changed files with 3982 additions and 19 deletions
Showing only changes of commit 8b87078478 - Show all commits

No files matched your search

+24 -1
View File
@@ -62,7 +62,8 @@ jobs:
~/.cargo/registry
~/.cargo/git
src/target
key: linux-musl-cargo-${{ hashFiles('src/Cargo.lock') }}
tools/migrate-mphf/target
key: linux-musl-cargo-${{ hashFiles('src/Cargo.lock', 'tools/migrate-mphf/Cargo.lock') }}
restore-keys: linux-musl-cargo-
- name: Build static binary
@@ -83,6 +84,28 @@ jobs:
-H "Authorization: token $GITEA_TOKEN" \
-F "attachment=@/tmp/dist/obikmer-linux-x86_64"
# One-shot converter for indexes built with ptr_hash 1.x. A standalone
# crate (own Cargo.lock/target, outside src/'s workspace), built after
# obikmer's own asset is already uploaded: it never blocks that one.
- name: Build migrate-mphf (static)
working-directory: tools/migrate-mphf
env:
PKG_CONFIG_ALLOW_CROSS: "1"
run: cargo zigbuild --release --target x86_64-unknown-linux-musl
- name: Upload migrate-mphf artifact
working-directory: tools/migrate-mphf
env:
GITEA_TOKEN: ${{ secrets.GITEATOKEN }}
RELEASE_ID: ${{ needs.create-release.outputs.release_id }}
run: |
cp target/x86_64-unknown-linux-musl/release/migrate-mphf /tmp/dist/migrate-mphf-linux-x86_64
strip /tmp/dist/migrate-mphf-linux-x86_64
curl -s -X POST \
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases/$RELEASE_ID/assets" \
-H "Authorization: token $GITEA_TOKEN" \
-F "attachment=@/tmp/dist/migrate-mphf-linux-x86_64"
build-macos-arm64:
needs: create-release
runs-on: ubuntu-latest
+1
View File
@@ -9,6 +9,7 @@ CLAUDE.md
memory/
sandbox/
src/target
tools/migrate-mphf/target
data-stress
*.fasta
*.fasta.gz
+12
View File
@@ -42,6 +42,18 @@ obikmer:
debug:
cargo build --manifest-path $(CARGO_DIR)/Cargo.toml --bin obikmer
# One-shot converter of indexes built with ptr_hash 1.x (see tools/migrate-mphf).
# Standalone crate, outside the src/ workspace, with its own Cargo.lock/target.
MIGRATE_MPHF_DIR := tools/migrate-mphf
.PHONY: migrate-mphf
migrate-mphf:
cargo build --release --manifest-path $(MIGRATE_MPHF_DIR)/Cargo.toml
.PHONY: clean-migrate-mphf
clean-migrate-mphf:
rm -rf $(MIGRATE_MPHF_DIR)/target
.PHONY: profile-build
profile-build:
cargo build --release --manifest-path $(CARGO_DIR)/Cargo.toml \
-2
View File
@@ -1528,7 +1528,6 @@ dependencies = [
name = "obikindexer"
version = "0.1.0"
dependencies = [
"cacheline-ef",
"epserde",
"memmap2",
"niffler",
@@ -1543,7 +1542,6 @@ dependencies = [
"obiskbuilder",
"obiskio",
"obisys",
"ptr_hash",
"rayon",
"serde",
"serde_json",
+25
View File
@@ -514,6 +514,31 @@ impl PersistentSparseBitMatrixBuilder {
}
Ok(builder)
}
/// Builds the row-permuted copy of `src` into `dir`: row `s` of the
/// result is row `new_to_old[s]` of `src`. `new_to_old` must have
/// `src.n()` entries and be a permutation of `0..src.n()` — the caller
/// proves that, this only reads `src` at those slots.
///
/// Needed whenever the slot numbering of a layer changes (its MPHF is
/// rebuilt) while the data attached to each k-mer must follow it.
/// Rows are read at arbitrary slots of `src`, so convert a dense source
/// to sparse first (`build_from_dense`) rather than permuting it
/// cell by cell.
pub fn build_permuted(
src: &PersistentSparseBitMatrix,
new_to_old: &[u32],
dir: &Path,
) -> io::Result<Self> {
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
let mut genomes: Vec<u32> = Vec::new();
for &old in new_to_old {
genomes.clear();
src.for_each_genome_in_row(old as usize, |g| genomes.push(g as u32));
builder.push_row(&genomes);
}
Ok(builder)
}
}
// ── Trait impls ───────────────────────────────────────────────────────────────
+23
View File
@@ -548,6 +548,29 @@ impl PersistentSparseCompactIntMatrixBuilder {
Ok(builder)
}
/// Row-permuted copy of `src`: row `s` of the result is row
/// `new_to_old[s]` of `src`. Same contract as
/// [`PersistentSparseBitMatrixBuilder::build_permuted`].
pub fn build_permuted(
src: &PersistentSparseCompactIntMatrix,
new_to_old: &[u32],
dir: &Path,
) -> io::Result<Self> {
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
let mut cols: Vec<u32> = Vec::new();
let mut vals: Vec<u32> = Vec::new();
for &old in new_to_old {
cols.clear();
vals.clear();
src.for_each_cell_in_row(old as usize, |c, v| {
cols.push(c as u32);
vals.push(v);
});
builder.push_row(&cols, &vals);
}
Ok(builder)
}
pub fn close(self) -> io::Result<()> {
let Self {
support,
+28
View File
@@ -415,3 +415,31 @@ fn partial_jaccard_and_hamming_match_dense_including_diagonal() {
drop(dir_dense);
drop(dir_sparse);
}
#[test]
fn build_permuted_follows_the_permutation() {
// Mix of singleton and multi rows, including a repeated multi set, so the
// dictionary dedup is exercised on the permuted order too.
let rows: Vec<&[bool]> = vec![
&[true, false, false, false],
&[false, false, true, false],
&[true, true, false, false],
&[false, false, true, true],
&[true, true, false, false],
];
let (_dir, src) = make_sparse(&rows, 4);
let new_to_old: Vec<u32> = vec![3, 0, 4, 1, 2];
let out = tempdir().unwrap();
let permuted =
PersistentSparseBitMatrixBuilder::build_permuted(&src, &new_to_old, &out.path().join("p"))
.unwrap()
.finish()
.unwrap();
assert_eq!(permuted.n(), src.n());
assert_eq!(permuted.n_cols(), src.n_cols());
for (new, &old) in new_to_old.iter().enumerate() {
assert_eq!(row_as_bool(&permuted, new), row_as_bool(&src, old as usize), "new row {new}");
}
}
@@ -304,3 +304,32 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
assert_eq!(&*sparse.row(slot), &**expected, "slot {slot}");
}
}
#[test]
fn build_permuted_follows_the_permutation() {
// Rows sharing a support with different values, plus singletons.
let rows: Vec<&[u32]> = vec![
&[7, 0, 0, 0],
&[0, 0, 3, 0],
&[2, 5, 0, 0],
&[9, 1, 0, 0],
&[0, 0, 4, 6],
];
let (_dir, src) = make_sparse(&rows, 4);
let new_to_old: Vec<u32> = vec![4, 2, 0, 3, 1];
let out = tempdir().unwrap();
let permuted = PersistentSparseCompactIntMatrixBuilder::build_permuted(
&src,
&new_to_old,
&out.path().join("p"),
)
.unwrap()
.finish()
.unwrap();
assert_eq!(permuted.n(), src.n());
for (new, &old) in new_to_old.iter().enumerate() {
assert_eq!(permuted.row(new), src.row(old as usize), "new row {new}");
}
}
+8
View File
@@ -210,6 +210,14 @@ impl KmerIndex {
crate::partition::index_dir(&self.root_path, i)
}
/// Path of layer `j` of partition `i` — pure path computation, nothing
/// is opened (unlike [`layer`](Self::layer)/`layer_dir`, which open the
/// layer's MPHF). For maintenance tools that must reach a layer's files
/// without going through a format they can't read yet.
pub fn layer_path(&self, i: usize, j: usize) -> PathBuf {
crate::layer::utils::layer_dir(&self.index_dir(i), j)
}
/// Number of layers in partition `i` — a disk scan, not a metadata read
/// (see [`KmerPartition::n_layers`](crate::partition::KmerPartition::n_layers)).
pub fn n_layers(&self, i: usize) -> OKIResult<usize> {
+3 -1
View File
@@ -9,5 +9,7 @@ pub(crate) mod utils;
pub use crate::index::error::{OKIError, OKIResult};
pub use content_layer::KmerLayer;
pub use meta::IndexMode;
pub use mphf_layer::{EvidenceKind, KmerBatchIter, KmerIter, MphfLayer, MphfOnly, UnitigIter};
pub use mphf_layer::{
EvidenceKind, KmerBatchIter, KmerIter, Mphf, MphfLayer, MphfOnly, UnitigIter, mphf_params,
};
pub use typed_layer::{HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer};
+9 -3
View File
@@ -23,7 +23,13 @@ pub(crate) const FINGERPRINT_FILE: &str = "fingerprint.bin";
/// The layer MPHF: `Xxh3Int` hasher, `CubicEps` buckets, multi-part (`SINGLE_PART = false`),
/// minimal (`REMAP = true`). Built owned (construction + store); queried through
/// `MemCase<Mphf>::uncase()`, the ε-copy view into the memory-mapped file.
pub(crate) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
pub type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
/// Construction parameters of [`Mphf`] — the one place they are chosen
/// (`lambda = 3.5`, `alpha = 0.99`, `CubicEps`, ~2.4 bits/key).
pub fn mphf_params() -> PtrHashParams<CubicEps> {
PtrHashParams::<CubicEps>::default_balanced()
}
// ── LayerEvidence ─────────────────────────────────────────────────────────────
@@ -550,7 +556,7 @@ impl MphfLayer {
// ── Empty layer ───────────────────────────────────────────────────────
if n == 0 {
let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::<CubicEps>::default_balanced())
let mphf: Mphf = Mphf::try_new(&[] as &[u64], mphf_params())
.ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?;
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
@@ -574,7 +580,7 @@ impl MphfLayer {
let mphf: Mphf = Mphf::new_from_par_iter(
n,
keys.map(|k| k.raw()).par_bridge(),
PtrHashParams::<CubicEps>::default_balanced(),
mphf_params(),
);
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
-2
View File
@@ -14,9 +14,7 @@ obicompactvec = { path = "../obicompactvec" }
obipipeline = { path = "../obipipeline" }
obiread = { path = "../obiread" }
obiskbuilder = { path = "../obiskbuilder" }
cacheline-ef = "2.0"
epserde = "0.13"
ptr_hash = { version = "2.1", features = ["epserde"] }
niffler = "3.0.0"
memmap2 = "0.9.10"
rayon = "1"
@@ -3,18 +3,15 @@ use std::fs;
use std::io;
use std::path::Path;
use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::ser::Serialize as EpSerialize;
use memmap2::Mmap;
use obicompactvec::PersistentCompactIntVecBuilder;
use obikindex::layer::{Mphf, mphf_params};
use obiskio::{SKFileReader, SKResult};
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int};
use tracing::debug;
use super::kmer_sort::sort_unique_kmers;
pub(super) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
let file = fs::File::open(unique_path)?;
let mmap = unsafe { Mmap::map(&file)? };
@@ -24,7 +21,7 @@ fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
// Sequential constructor: the outer par_iter over partitions already saturates
// the Rayon pool. new_from_par_iter would get no additional threads and adds
// coordination overhead. try_new accesses the same mmap'd pages at zero extra cost.
Mphf::try_new(kmers, PtrHashParams::<CubicEps>::default_balanced())
Mphf::try_new(kmers, mphf_params())
.ok_or_else(|| io::Error::other("ptr_hash construction failed"))
}
@@ -25,17 +25,13 @@ use std::io;
use std::path::Path;
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::prelude::*;
use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{KmerLayer, TypedLayer};
use obikindex::layer::{KmerLayer, Mphf, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xxh3Int};
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
pub(crate) trait PrivateBuilder {
/// Mark scatter as complete (`IndexState::Scattered`).
+3216
View File
File diff suppressed because it is too large. Load diff
+36
View File
@@ -0,0 +1,36 @@
[package]
name = "migrate-mphf"
version = "0.1.0"
edition = "2024"
publish = false
description = "One-shot converter: obikmer indexes whose layer MPHFs were built with ptr_hash 1.x -> ptr_hash 2.x"
# Standalone: not a member of src/'s workspace, own Cargo.lock. The previous
# generation of the MPHF stack (and its `proc-macro-error2` future-incompat
# warning) lives only here, never in obikmer itself.
[workspace]
[features]
default = []
numa = ["obisys/numa"]
[dependencies]
obikindex = { path = "../../src/obikindex" }
obicompactvec = { path = "../../src/obicompactvec" }
obiskio = { path = "../../src/obiskio" }
obisys = { path = "../../src/obisys", default-features = false }
epserde = "0.13"
rayon = "1"
anyhow = "1"
clap = { version = "4", features = ["derive"] }
tracing = "0.1.44"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
# The generation the indexes to convert were built with, under other names
# so that both can live in this one binary.
ptr_hash_old = { package = "ptr_hash", version = "=1.1.0" }
epserde_old = { package = "epserde", version = "=0.8.0" }
cacheline_ef_old = { package = "cacheline-ef", version = "=1.1.0" }
[profile.release]
debug = 1
+98
View File
@@ -0,0 +1,98 @@
//! Crash-safe replacement of a layer's files by their converted versions.
//!
//! Everything new is written under `<layer>/.migrate_mphf/` first; nothing of
//! the layer is touched until that is complete and verified. Then a `COMMIT`
//! marker is written, and [`finish`] moves the staged files into place. Every
//! step of [`finish`] is idempotent ("if the staged item still exists, move
//! it"), so a layer interrupted after `COMMIT` is simply finished by the next
//! run. Files are only ever replaced by `rename`, never written in place:
//! some of them are hard-linked from elsewhere (`ls -l` link count 2), and an
//! in-place write would corrupt the other link.
use std::fs;
use std::io;
use std::path::Path;
use anyhow::{Context, Result};
pub const STAGING: &str = ".migrate_mphf";
pub const COMMIT: &str = "COMMIT";
/// Written last: the layer is converted, skip it on later runs.
pub const DONE: &str = ".mphf_migrated";
/// Where derived annexes go instead of being deleted.
pub const STALE_DIR: &str = "stale_annexes_pre_ptrhash2";
/// Replaced as directories.
const DIRS: [&str; 2] = ["presence", "counts"];
/// Replaced as files; `mphf.bin` last.
const FILES: [&str; 3] = ["evidence.bin", "fingerprint.bin", "mphf.bin"];
/// Slot-ordered or slot-derived annexes of a layer: wrong (silently) once the
/// slot numbering changes, so they are set aside, to be rebuilt.
const LAYER_ANNEXES: [&str; 7] = [
"siblings.psib",
"entropy.pent",
"entropy.noamb.pent",
"pairweight.ppwt",
"pairweight.noamb.ppwt",
"quartet.pbiv",
"quartet.noamb.pbiv",
];
/// Index-level annexes derived from the sibling annex.
pub const INDEX_ANNEXES: [&str; 2] = ["family_overlap.povl", "family_overlap.noamb.povl"];
/// Move the staged files into `dir`, set the stale annexes aside, mark the
/// layer done and drop the staging area.
pub fn finish(dir: &Path) -> Result<()> {
let staging = dir.join(STAGING);
for name in DIRS {
let staged = staging.join(name);
if staged.exists() {
let target = dir.join(name);
if target.exists() {
fs::remove_dir_all(&target).with_context(|| format!("removing {}", target.display()))?;
}
fs::rename(&staged, &target).with_context(|| format!("installing {}", target.display()))?;
}
}
for name in FILES {
let staged = staging.join(name);
if staged.exists() {
let target = dir.join(name);
fs::rename(&staged, &target).with_context(|| format!("installing {}", target.display()))?;
}
}
set_aside(dir, &LAYER_ANNEXES)?;
fs::write(dir.join(DONE), env!("CARGO_PKG_VERSION"))?;
fs::remove_dir_all(&staging)?;
Ok(())
}
/// Move each of `names` that exists in `dir` into `dir/STALE_DIR/`.
/// Returns how many were moved.
pub fn set_aside(dir: &Path, names: &[&str]) -> Result<usize> {
let aside = dir.join(STALE_DIR);
let mut moved = 0;
for name in names {
let from = dir.join(name);
if !from.exists() {
continue;
}
fs::create_dir_all(&aside)?;
fs::rename(&from, aside.join(name)).with_context(|| format!("setting aside {}", from.display()))?;
moved += 1;
}
Ok(moved)
}
/// Remove a staging area left by an interrupted, uncommitted run.
pub fn discard(dir: &Path) -> io::Result<()> {
match fs::remove_dir_all(dir.join(STAGING)) {
Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e),
_ => Ok(()),
}
}
+100
View File
@@ -0,0 +1,100 @@
//! Permute the per-slot files of a layer into `staging/`.
//!
//! Presence and count matrices always come out in the compact row-major
//! (sparse) format, whatever they were: a dense source is first converted
//! sequentially (`build_from_dense`), then permuted from the sparse form,
//! whose rows are cheap to read at arbitrary slots.
use std::fs;
use std::path::Path;
use anyhow::{Result, ensure};
use obicompactvec::{
PersistentBitMatrix, PersistentIntMatrix, PersistentSparseBitMatrixBuilder,
PersistentSparseCompactIntMatrixBuilder,
};
use obikindex::layer::evidence::{Evidence, EvidenceWriter};
use obikindex::layer::fingerprint::{FingerprintVec, FingerprintVecWriter};
use crate::plan::Plan;
use crate::util::oki;
/// Every per-slot file of the layer in `dir` that exists, permuted into `staging`.
pub fn permute_all(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
if dir.join("evidence.bin").exists() {
evidence(dir, staging, plan)?;
}
if dir.join("fingerprint.bin").exists() {
fingerprint(dir, staging, plan)?;
}
if dir.join("presence").is_dir() {
presence(dir, staging, plan)?;
}
if dir.join("counts").is_dir() {
counts(dir, staging, plan)?;
}
Ok(())
}
fn evidence(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
let old = oki(Evidence::open(&dir.join("evidence.bin")))?;
ensure!(old.len() == plan.n, "evidence.bin has {} slots, layer has {}", old.len(), plan.n);
let mut out = EvidenceWriter::new(plan.n);
for (s, &o) in plan.new_to_old.iter().enumerate() {
let (chunk_id, rank) = old.decode(o as usize);
out.set(s, chunk_id, rank);
}
oki(out.write(&staging.join("evidence.bin")))
}
fn fingerprint(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
let old = oki(FingerprintVec::open(&dir.join("fingerprint.bin")))?;
ensure!(old.n() == plan.n, "fingerprint.bin has {} slots, layer has {}", old.n(), plan.n);
let mut out = FingerprintVecWriter::new(plan.n, old.b());
for (s, &o) in plan.new_to_old.iter().enumerate() {
out.set(s, old.get(o as usize));
}
oki(out.write(&staging.join("fingerprint.bin")))
}
fn presence(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
let out = staging.join("presence");
let src = match PersistentBitMatrix::open(dir)? {
// Mono-genome: no file, nothing is attached to slots.
PersistentBitMatrix::Implicit { .. } => return Ok(()),
PersistentBitMatrix::Sparse(s) => {
ensure!(s.n() == plan.n, "presence has {} rows, layer has {}", s.n(), plan.n);
PersistentSparseBitMatrixBuilder::build_permuted(&s, &plan.new_to_old, &out)?.close()?;
return Ok(());
}
dense => dense,
};
ensure!(src.n() == plan.n, "presence has {} rows, layer has {}", src.n(), plan.n);
let tmp = staging.join("presence_dense_to_sparse");
let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&src, &tmp)?.finish()?;
drop(src);
PersistentSparseBitMatrixBuilder::build_permuted(&sparse, &plan.new_to_old, &out)?.close()?;
drop(sparse);
fs::remove_dir_all(&tmp)?;
Ok(())
}
fn counts(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
let out = staging.join("counts");
let src = match PersistentIntMatrix::open(dir)? {
PersistentIntMatrix::Sparse(s) => {
ensure!(s.n() == plan.n, "counts has {} rows, layer has {}", s.n(), plan.n);
PersistentSparseCompactIntMatrixBuilder::build_permuted(&s, &plan.new_to_old, &out)?.close()?;
return Ok(());
}
dense => dense,
};
ensure!(src.n() == plan.n, "counts has {} rows, layer has {}", src.n(), plan.n);
let tmp = staging.join("counts_dense_to_sparse");
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&src, &tmp)?.finish()?;
drop(src);
PersistentSparseCompactIntMatrixBuilder::build_permuted(&sparse, &plan.new_to_old, &out)?.close()?;
drop(sparse);
fs::remove_dir_all(&tmp)?;
Ok(())
}
+75
View File
@@ -0,0 +1,75 @@
//! Convert one layer directory: new MPHF, permuted per-slot data, verified,
//! then installed atomically (see `commit`).
use std::fs;
use std::path::Path;
use anyhow::{Result, anyhow, ensure};
use epserde::prelude::*;
use obikindex::layer::{Mphf, mphf_params};
use obiskio::UnitigFileReader;
use crate::util::sk;
use crate::{commit, data, old, plan, verify};
pub struct Options {
/// How many k-mers per layer to compare old against new (ignored with `verify_full`).
pub verify_sample: usize,
pub verify_full: bool,
/// Do everything except installing the result.
pub dry_run: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Outcome {
/// Already converted by an earlier run.
Skipped,
/// Interrupted after verification by an earlier run; installed now.
Resumed,
Converted { n: usize, verified: usize },
/// Everything but the installation.
Checked { n: usize, verified: usize },
}
pub fn convert(dir: &Path, opts: &Options) -> Result<Outcome> {
if dir.join(commit::DONE).exists() {
commit::discard(dir)?;
return Ok(Outcome::Skipped);
}
let staging = dir.join(commit::STAGING);
if staging.join(commit::COMMIT).exists() {
commit::finish(dir)?;
return Ok(Outcome::Resumed);
}
commit::discard(dir)?;
fs::create_dir_all(&staging)?;
let n = sk(UnitigFileReader::open_sequential(&dir.join("unitigs.bin")))?.n_kmers();
ensure!(n <= u32::MAX as usize, "layer has {n} k-mers, more than the 2^32 this tool handles");
let verified = if n == 0 {
empty_mphf(&staging)?;
0
} else {
let old_mphf = old::load(&dir.join("mphf.bin"))?;
let plan = plan::build(dir, &staging, n, &old_mphf)?;
data::permute_all(dir, &staging, &plan)?;
let stride = if opts.verify_full { 1 } else { (n / opts.verify_sample.max(1)).max(1) };
verify::check(dir, &staging, &plan, &old_mphf, stride)?
};
if opts.dry_run {
commit::discard(dir)?;
return Ok(Outcome::Checked { n, verified });
}
fs::write(staging.join(commit::COMMIT), b"")?;
commit::finish(dir)?;
Ok(Outcome::Converted { n, verified })
}
/// An empty layer still carries an MPHF (of nothing), like `MphfLayer::build` writes.
fn empty_mphf(staging: &Path) -> Result<()> {
let mphf = Mphf::try_new(&[] as &[u64], mphf_params())
.ok_or_else(|| anyhow!("empty MPHF construction failed"))?;
unsafe { mphf.store(staging.join("mphf.bin")) }.map_err(|e| anyhow!("storing the empty MPHF: {e}"))
}
+121
View File
@@ -0,0 +1,121 @@
//! One-shot converter for obikmer indexes whose layer MPHFs were built with
//! `ptr_hash` 1.x: rebuilds each MPHF with `ptr_hash` 2.x and carries the
//! per-slot data (evidence, fingerprint, presence/count matrices) over to the
//! new slot numbering, in place, layer by layer, verified before installing.
//!
//! Presence and count matrices come out in the compact row-major format.
//! Annexes derived from slot numbering (sibling, entropy, pair-weight,
//! quartet, family-overlap) are set aside under `stale_annexes_pre_ptrhash2/`
//! and must be rebuilt.
mod commit;
mod data;
mod layer;
mod old;
mod plan;
mod util;
mod verify;
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use anyhow::{Result, ensure};
use clap::Parser;
use obikindex::KmerIndex;
use obisys::PartitionRunner;
use tracing::{info, warn};
use layer::{Options, Outcome};
use util::oki;
#[derive(Parser)]
#[command(about = "Convert an obikmer index from ptr_hash 1.x to ptr_hash 2.x MPHFs, in place")]
struct Cli {
/// Index directory (the one holding `index.meta` and `part_*`).
index: PathBuf,
/// Limit the number of CPUs used.
#[arg(long)]
cpu_max: Option<usize>,
/// K-mers compared old-vs-new per layer before installing it.
#[arg(long, default_value_t = 200_000)]
verify_sample: usize,
/// Compare every k-mer of every layer (slow).
#[arg(long)]
verify_full: bool,
/// Do everything except installing: convert into staging, verify, discard.
#[arg(long)]
dry_run: bool,
/// Only these partitions (repeatable). The index-level annexes are then left alone.
#[arg(long)]
partition: Vec<usize>,
/// Only these layers of each partition (repeatable). The index-level annexes are then left alone.
#[arg(long)]
layer: Vec<usize>,
}
fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()))
.init();
let cli = Cli::parse();
if let Some(n) = cli.cpu_max {
obisys::set_cpu_cap(n);
}
// Holds the index's directory lock for the whole run, and sets k.
let index = oki(KmerIndex::open_lock(&cli.index))?;
let n_partitions = index.n_partitions();
let partial = !cli.partition.is_empty() || !cli.layer.is_empty();
let order: Vec<usize> = if cli.partition.is_empty() { (0..n_partitions).collect() } else { cli.partition.clone() };
ensure!(order.iter().all(|&p| p < n_partitions), "index has only {n_partitions} partitions");
let opts = Options { verify_sample: cli.verify_sample, verify_full: cli.verify_full, dry_run: cli.dry_run };
let converted = AtomicUsize::new(0);
let skipped = AtomicUsize::new(0);
let mut done = 0usize;
PartitionRunner::new().run(
&order,
|p| -> Result<Vec<(usize, Outcome)>> {
let n_layers = oki(index.n_layers(p))?;
let mut outcomes = Vec::new();
for l in (0..n_layers).filter(|l| cli.layer.is_empty() || cli.layer.contains(l)) {
let dir = index.layer_path(p, l);
let outcome = layer::convert(&dir, &opts)
.map_err(|e| e.context(format!("partition {p}, layer {l} ({})", dir.display())))?;
match outcome {
Outcome::Skipped => skipped.fetch_add(1, Ordering::Relaxed),
_ => converted.fetch_add(1, Ordering::Relaxed),
};
outcomes.push((l, outcome));
}
Ok(outcomes)
},
|p, outcomes, elapsed| {
done += 1;
info!("partition {p} ({done}/{}) in {:.1?}: {outcomes:?}", order.len(), elapsed);
},
)?;
if !cli.dry_run && !partial {
let moved = commit::set_aside(index.dir(), &commit::INDEX_ANNEXES)?;
if moved > 0 {
info!("{moved} index-level annex(es) set aside under {}", commit::STALE_DIR);
}
} else if !cli.dry_run {
warn!("partial run: index-level annexes (family overlap) not set aside; do it on the final pass");
}
info!(
"done: {} layer(s) {}, {} already converted",
converted.load(Ordering::Relaxed),
if cli.dry_run { "checked" } else { "converted" },
skipped.load(Ordering::Relaxed)
);
Ok(())
}
+17
View File
@@ -0,0 +1,17 @@
//! The previous MPHF generation (`ptr_hash` 1.x), kept only to read the
//! `mphf.bin` files of indexes built before the migration.
use std::path::Path;
use anyhow::{Result, anyhow};
use cacheline_ef_old::{CachelineEf, CachelineEfVec};
use epserde_old::prelude::*;
use ptr_hash_old::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
/// Exactly the type `obikindex` used before the migration.
pub type OldMphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
pub fn load(path: &Path) -> Result<OldMphf> {
OldMphf::load_full(path)
.map_err(|e| anyhow!("{}: not a ptr_hash 1.x MPHF (already converted?): {e}", path.display()))
}
+53
View File
@@ -0,0 +1,53 @@
//! The slot permutation between a layer's old and new MPHF.
//!
//! A layer's MPHF gives each k-mer a slot, and every other per-slot file of
//! the layer (evidence, fingerprint, presence/count matrices) is indexed by
//! it. A new MPHF numbers the same k-mers differently, so the data has to
//! follow: the k-mers themselves come from `unitigs.bin`, which does not
//! depend on the MPHF at all.
use std::path::Path;
use anyhow::{Result, anyhow, ensure};
use epserde::prelude::*;
use obikindex::layer::{Mphf, mphf_params};
use obiskio::CanonicalKmerIter;
use rayon::prelude::*;
use crate::old::OldMphf;
use crate::util::sk;
/// `new_to_old[s]` is the old slot of the k-mer that the new MPHF puts in
/// slot `s`. Always a bijection on `0..n` (checked while it is built).
pub struct Plan {
pub n: usize,
pub new_to_old: Vec<u32>,
}
/// Build the new MPHF of the layer whose `unitigs.bin` is in `dir`, store it
/// as `staging/mphf.bin`, and derive the permutation against `old`.
pub fn build(dir: &Path, staging: &Path, n: usize, old: &OldMphf) -> Result<Plan> {
let unitigs = dir.join("unitigs.bin");
// Same construction as `MphfLayer::build`: a clonable mmap iterator, bridged.
let keys = sk(CanonicalKmerIter::new(&unitigs))?;
let new: Mphf = Mphf::new_from_par_iter(n, keys.map(|k| k.raw()).par_bridge(), mphf_params());
unsafe { new.store(staging.join("mphf.bin")) }.map_err(|e| anyhow!("storing the new MPHF: {e}"))?;
let mut new_to_old = vec![u32::MAX; n];
let mut old_seen = vec![0u8; n.div_ceil(8)];
for kmer in sk(CanonicalKmerIter::new(&unitigs))? {
let key = kmer.raw();
let o = old.index(&key);
let s = new.index(&key);
ensure!(o < n, "old MPHF: slot {o} out of range (n = {n}) — is this layer's mphf.bin really the one of its unitigs.bin?");
ensure!(s < n, "new MPHF: slot {s} out of range (n = {n})");
ensure!(new_to_old[s] == u32::MAX, "new MPHF puts two k-mers in slot {s}");
let (byte, bit) = (o / 8, 1u8 << (o % 8));
ensure!(old_seen[byte] & bit == 0, "old MPHF puts two k-mers in slot {o}");
old_seen[byte] |= bit;
new_to_old[s] = o as u32;
}
// n distinct new slots, all in range, among n: the map is total.
Ok(Plan { n, new_to_old })
}
+11
View File
@@ -0,0 +1,11 @@
//! Error adapters: the libraries each have their own error type.
use anyhow::{Result, anyhow};
pub fn sk<T>(r: obiskio::SKResult<T>) -> Result<T> {
r.map_err(|e| anyhow!("{e}"))
}
pub fn oki<T>(r: obikindex::OKIResult<T>) -> Result<T> {
r.map_err(|e| anyhow!("{e}"))
}
+90
View File
@@ -0,0 +1,90 @@
//! Check the staged files against the layer as it still is, before anything
//! is replaced.
//!
//! For a sample of k-mers read from `unitigs.bin`, the old chain (old MPHF →
//! old slot → old data) must agree with the new one (staged MPHF, loaded the
//! way queries load it → new slot → staged data): same evidence, same
//! fingerprint, same presence set, same counts. A full check is the same loop
//! with every k-mer.
use std::path::Path;
use anyhow::{Result, anyhow, ensure};
use epserde::prelude::*;
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseCompactIntMatrix};
use obikindex::layer::Mphf;
use obikindex::layer::evidence::Evidence;
use obikindex::layer::fingerprint::FingerprintVec;
use obiskio::CanonicalKmerIter;
use crate::old::OldMphf;
use crate::plan::Plan;
use crate::util::{oki, sk};
/// Both sides of one per-slot file, when the layer has it.
struct Pair<O, N> {
old: O,
new: N,
}
fn pair<O, N>(
exists: bool,
old: impl FnOnce() -> Result<O>,
new: impl FnOnce() -> Result<N>,
) -> Result<Option<Pair<O, N>>> {
Ok(if exists { Some(Pair { old: old()?, new: new()? }) } else { None })
}
/// Returns how many k-mers were compared.
pub fn check(dir: &Path, staging: &Path, plan: &Plan, old_mphf: &OldMphf, stride: usize) -> Result<usize> {
let staged_mphf = unsafe { Mphf::mmap(staging.join("mphf.bin"), Flags::empty()) }
.map_err(|e| anyhow!("staged mphf.bin does not load: {e}"))?;
let new_mphf = staged_mphf.uncase();
let evidence = pair(
staging.join("evidence.bin").exists(),
|| oki(Evidence::open(&dir.join("evidence.bin"))),
|| oki(Evidence::open(&staging.join("evidence.bin"))),
)?;
let fingerprint = pair(
staging.join("fingerprint.bin").exists(),
|| oki(FingerprintVec::open(&dir.join("fingerprint.bin"))),
|| oki(FingerprintVec::open(&staging.join("fingerprint.bin"))),
)?;
let presence = pair(
staging.join("presence").is_dir(),
|| Ok(PersistentBitMatrix::open(dir)?),
|| Ok(PersistentSparseBitMatrix::open(&staging.join("presence"))?),
)?;
let counts = pair(
staging.join("counts").is_dir(),
|| Ok(PersistentIntMatrix::open(dir)?),
|| Ok(PersistentSparseCompactIntMatrix::open(&staging.join("counts"))?),
)?;
let mut compared = 0;
for (i, kmer) in sk(CanonicalKmerIter::new(&dir.join("unitigs.bin")))?.enumerate() {
if i % stride != 0 {
continue;
}
let key = kmer.raw();
let o = old_mphf.index(&key);
let s = new_mphf.index(&key);
ensure!(s < plan.n && plan.new_to_old[s] as usize == o, "k-mer #{i}: staged MPHF slot {s} disagrees with the plan (old slot {o})");
if let Some(p) = &evidence {
ensure!(p.old.decode(o) == p.new.decode(s), "k-mer #{i}: evidence differs (old slot {o}, new slot {s})");
}
if let Some(p) = &fingerprint {
ensure!(p.old.get(o) == p.new.get(s), "k-mer #{i}: fingerprint differs (old slot {o}, new slot {s})");
}
if let Some(p) = &presence {
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: presence row differs (old slot {o}, new slot {s})");
}
if let Some(p) = &counts {
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: count row differs (old slot {o}, new slot {s})");
}
compared += 1;
}
Ok(compared)
}