Implement MPHF migration tool and sparse matrix permutation logic
Refactors sparse matrix handling to implement row permutation logic and introduces a standalone tool for migrating indexes from MPHF 1.x to 2.x formats.
This commit is contained in:
1 parent
d686d39506
commit
8b87078478
24 files changed
+3982
-19
No files matched your search
@@ -62,7 +62,8 @@ jobs:
|
|||||||
~/.cargo/registry
|
~/.cargo/registry
|
||||||
~/.cargo/git
|
~/.cargo/git
|
||||||
src/target
|
src/target
|
||||||
key: linux-musl-cargo-${{ hashFiles('src/Cargo.lock') }}
|
tools/migrate-mphf/target
|
||||||
|
key: linux-musl-cargo-${{ hashFiles('src/Cargo.lock', 'tools/migrate-mphf/Cargo.lock') }}
|
||||||
restore-keys: linux-musl-cargo-
|
restore-keys: linux-musl-cargo-
|
||||||
|
|
||||||
- name: Build static binary
|
- name: Build static binary
|
||||||
@@ -83,6 +84,28 @@ jobs:
|
|||||||
-H "Authorization: token $GITEA_TOKEN" \
|
-H "Authorization: token $GITEA_TOKEN" \
|
||||||
-F "attachment=@/tmp/dist/obikmer-linux-x86_64"
|
-F "attachment=@/tmp/dist/obikmer-linux-x86_64"
|
||||||
|
|
||||||
|
# One-shot converter for indexes built with ptr_hash 1.x. A standalone
|
||||||
|
# crate (own Cargo.lock/target, outside src/'s workspace), built after
|
||||||
|
# obikmer's own asset is already uploaded: it never blocks that one.
|
||||||
|
- name: Build migrate-mphf (static)
|
||||||
|
working-directory: tools/migrate-mphf
|
||||||
|
env:
|
||||||
|
PKG_CONFIG_ALLOW_CROSS: "1"
|
||||||
|
run: cargo zigbuild --release --target x86_64-unknown-linux-musl
|
||||||
|
|
||||||
|
- name: Upload migrate-mphf artifact
|
||||||
|
working-directory: tools/migrate-mphf
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEATOKEN }}
|
||||||
|
RELEASE_ID: ${{ needs.create-release.outputs.release_id }}
|
||||||
|
run: |
|
||||||
|
cp target/x86_64-unknown-linux-musl/release/migrate-mphf /tmp/dist/migrate-mphf-linux-x86_64
|
||||||
|
strip /tmp/dist/migrate-mphf-linux-x86_64
|
||||||
|
curl -s -X POST \
|
||||||
|
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases/$RELEASE_ID/assets" \
|
||||||
|
-H "Authorization: token $GITEA_TOKEN" \
|
||||||
|
-F "attachment=@/tmp/dist/migrate-mphf-linux-x86_64"
|
||||||
|
|
||||||
build-macos-arm64:
|
build-macos-arm64:
|
||||||
needs: create-release
|
needs: create-release
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ CLAUDE.md
|
|||||||
memory/
|
memory/
|
||||||
sandbox/
|
sandbox/
|
||||||
src/target
|
src/target
|
||||||
|
tools/migrate-mphf/target
|
||||||
data-stress
|
data-stress
|
||||||
*.fasta
|
*.fasta
|
||||||
*.fasta.gz
|
*.fasta.gz
|
||||||
|
|||||||
@@ -42,6 +42,18 @@ obikmer:
|
|||||||
debug:
|
debug:
|
||||||
cargo build --manifest-path $(CARGO_DIR)/Cargo.toml --bin obikmer
|
cargo build --manifest-path $(CARGO_DIR)/Cargo.toml --bin obikmer
|
||||||
|
|
||||||
|
# One-shot converter of indexes built with ptr_hash 1.x (see tools/migrate-mphf).
|
||||||
|
# Standalone crate, outside the src/ workspace, with its own Cargo.lock/target.
|
||||||
|
MIGRATE_MPHF_DIR := tools/migrate-mphf
|
||||||
|
|
||||||
|
.PHONY: migrate-mphf
|
||||||
|
migrate-mphf:
|
||||||
|
cargo build --release --manifest-path $(MIGRATE_MPHF_DIR)/Cargo.toml
|
||||||
|
|
||||||
|
.PHONY: clean-migrate-mphf
|
||||||
|
clean-migrate-mphf:
|
||||||
|
rm -rf $(MIGRATE_MPHF_DIR)/target
|
||||||
|
|
||||||
.PHONY: profile-build
|
.PHONY: profile-build
|
||||||
profile-build:
|
profile-build:
|
||||||
cargo build --release --manifest-path $(CARGO_DIR)/Cargo.toml \
|
cargo build --release --manifest-path $(CARGO_DIR)/Cargo.toml \
|
||||||
|
|||||||
Generated
-2
@@ -1528,7 +1528,6 @@ dependencies = [
|
|||||||
name = "obikindexer"
|
name = "obikindexer"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cacheline-ef",
|
|
||||||
"epserde",
|
"epserde",
|
||||||
"memmap2",
|
"memmap2",
|
||||||
"niffler",
|
"niffler",
|
||||||
@@ -1543,7 +1542,6 @@ dependencies = [
|
|||||||
"obiskbuilder",
|
"obiskbuilder",
|
||||||
"obiskio",
|
"obiskio",
|
||||||
"obisys",
|
"obisys",
|
||||||
"ptr_hash",
|
|
||||||
"rayon",
|
"rayon",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
|||||||
@@ -514,6 +514,31 @@ impl PersistentSparseBitMatrixBuilder {
|
|||||||
}
|
}
|
||||||
Ok(builder)
|
Ok(builder)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Builds the row-permuted copy of `src` into `dir`: row `s` of the
|
||||||
|
/// result is row `new_to_old[s]` of `src`. `new_to_old` must have
|
||||||
|
/// `src.n()` entries and be a permutation of `0..src.n()` — the caller
|
||||||
|
/// proves that, this only reads `src` at those slots.
|
||||||
|
///
|
||||||
|
/// Needed whenever the slot numbering of a layer changes (its MPHF is
|
||||||
|
/// rebuilt) while the data attached to each k-mer must follow it.
|
||||||
|
/// Rows are read at arbitrary slots of `src`, so convert a dense source
|
||||||
|
/// to sparse first (`build_from_dense`) rather than permuting it
|
||||||
|
/// cell by cell.
|
||||||
|
pub fn build_permuted(
|
||||||
|
src: &PersistentSparseBitMatrix,
|
||||||
|
new_to_old: &[u32],
|
||||||
|
dir: &Path,
|
||||||
|
) -> io::Result<Self> {
|
||||||
|
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
|
||||||
|
let mut genomes: Vec<u32> = Vec::new();
|
||||||
|
for &old in new_to_old {
|
||||||
|
genomes.clear();
|
||||||
|
src.for_each_genome_in_row(old as usize, |g| genomes.push(g as u32));
|
||||||
|
builder.push_row(&genomes);
|
||||||
|
}
|
||||||
|
Ok(builder)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Trait impls ───────────────────────────────────────────────────────────────
|
// ── Trait impls ───────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -548,6 +548,29 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
|||||||
Ok(builder)
|
Ok(builder)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Row-permuted copy of `src`: row `s` of the result is row
|
||||||
|
/// `new_to_old[s]` of `src`. Same contract as
|
||||||
|
/// [`PersistentSparseBitMatrixBuilder::build_permuted`].
|
||||||
|
pub fn build_permuted(
|
||||||
|
src: &PersistentSparseCompactIntMatrix,
|
||||||
|
new_to_old: &[u32],
|
||||||
|
dir: &Path,
|
||||||
|
) -> io::Result<Self> {
|
||||||
|
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
|
||||||
|
let mut cols: Vec<u32> = Vec::new();
|
||||||
|
let mut vals: Vec<u32> = Vec::new();
|
||||||
|
for &old in new_to_old {
|
||||||
|
cols.clear();
|
||||||
|
vals.clear();
|
||||||
|
src.for_each_cell_in_row(old as usize, |c, v| {
|
||||||
|
cols.push(c as u32);
|
||||||
|
vals.push(v);
|
||||||
|
});
|
||||||
|
builder.push_row(&cols, &vals);
|
||||||
|
}
|
||||||
|
Ok(builder)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn close(self) -> io::Result<()> {
|
pub fn close(self) -> io::Result<()> {
|
||||||
let Self {
|
let Self {
|
||||||
support,
|
support,
|
||||||
|
|||||||
@@ -415,3 +415,31 @@ fn partial_jaccard_and_hamming_match_dense_including_diagonal() {
|
|||||||
drop(dir_dense);
|
drop(dir_dense);
|
||||||
drop(dir_sparse);
|
drop(dir_sparse);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_permuted_follows_the_permutation() {
|
||||||
|
// Mix of singleton and multi rows, including a repeated multi set, so the
|
||||||
|
// dictionary dedup is exercised on the permuted order too.
|
||||||
|
let rows: Vec<&[bool]> = vec![
|
||||||
|
&[true, false, false, false],
|
||||||
|
&[false, false, true, false],
|
||||||
|
&[true, true, false, false],
|
||||||
|
&[false, false, true, true],
|
||||||
|
&[true, true, false, false],
|
||||||
|
];
|
||||||
|
let (_dir, src) = make_sparse(&rows, 4);
|
||||||
|
let new_to_old: Vec<u32> = vec![3, 0, 4, 1, 2];
|
||||||
|
|
||||||
|
let out = tempdir().unwrap();
|
||||||
|
let permuted =
|
||||||
|
PersistentSparseBitMatrixBuilder::build_permuted(&src, &new_to_old, &out.path().join("p"))
|
||||||
|
.unwrap()
|
||||||
|
.finish()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(permuted.n(), src.n());
|
||||||
|
assert_eq!(permuted.n_cols(), src.n_cols());
|
||||||
|
for (new, &old) in new_to_old.iter().enumerate() {
|
||||||
|
assert_eq!(row_as_bool(&permuted, new), row_as_bool(&src, old as usize), "new row {new}");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -304,3 +304,32 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
|
|||||||
assert_eq!(&*sparse.row(slot), &**expected, "slot {slot}");
|
assert_eq!(&*sparse.row(slot), &**expected, "slot {slot}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_permuted_follows_the_permutation() {
|
||||||
|
// Rows sharing a support with different values, plus singletons.
|
||||||
|
let rows: Vec<&[u32]> = vec![
|
||||||
|
&[7, 0, 0, 0],
|
||||||
|
&[0, 0, 3, 0],
|
||||||
|
&[2, 5, 0, 0],
|
||||||
|
&[9, 1, 0, 0],
|
||||||
|
&[0, 0, 4, 6],
|
||||||
|
];
|
||||||
|
let (_dir, src) = make_sparse(&rows, 4);
|
||||||
|
let new_to_old: Vec<u32> = vec![4, 2, 0, 3, 1];
|
||||||
|
|
||||||
|
let out = tempdir().unwrap();
|
||||||
|
let permuted = PersistentSparseCompactIntMatrixBuilder::build_permuted(
|
||||||
|
&src,
|
||||||
|
&new_to_old,
|
||||||
|
&out.path().join("p"),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.finish()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(permuted.n(), src.n());
|
||||||
|
for (new, &old) in new_to_old.iter().enumerate() {
|
||||||
|
assert_eq!(permuted.row(new), src.row(old as usize), "new row {new}");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -210,6 +210,14 @@ impl KmerIndex {
|
|||||||
crate::partition::index_dir(&self.root_path, i)
|
crate::partition::index_dir(&self.root_path, i)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Path of layer `j` of partition `i` — pure path computation, nothing
|
||||||
|
/// is opened (unlike [`layer`](Self::layer)/`layer_dir`, which open the
|
||||||
|
/// layer's MPHF). For maintenance tools that must reach a layer's files
|
||||||
|
/// without going through a format they can't read yet.
|
||||||
|
pub fn layer_path(&self, i: usize, j: usize) -> PathBuf {
|
||||||
|
crate::layer::utils::layer_dir(&self.index_dir(i), j)
|
||||||
|
}
|
||||||
|
|
||||||
/// Number of layers in partition `i` — a disk scan, not a metadata read
|
/// Number of layers in partition `i` — a disk scan, not a metadata read
|
||||||
/// (see [`KmerPartition::n_layers`](crate::partition::KmerPartition::n_layers)).
|
/// (see [`KmerPartition::n_layers`](crate::partition::KmerPartition::n_layers)).
|
||||||
pub fn n_layers(&self, i: usize) -> OKIResult<usize> {
|
pub fn n_layers(&self, i: usize) -> OKIResult<usize> {
|
||||||
|
|||||||
@@ -9,5 +9,7 @@ pub(crate) mod utils;
|
|||||||
pub use crate::index::error::{OKIError, OKIResult};
|
pub use crate::index::error::{OKIError, OKIResult};
|
||||||
pub use content_layer::KmerLayer;
|
pub use content_layer::KmerLayer;
|
||||||
pub use meta::IndexMode;
|
pub use meta::IndexMode;
|
||||||
pub use mphf_layer::{EvidenceKind, KmerBatchIter, KmerIter, MphfLayer, MphfOnly, UnitigIter};
|
pub use mphf_layer::{
|
||||||
|
EvidenceKind, KmerBatchIter, KmerIter, Mphf, MphfLayer, MphfOnly, UnitigIter, mphf_params,
|
||||||
|
};
|
||||||
pub use typed_layer::{HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer};
|
pub use typed_layer::{HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer};
|
||||||
@@ -23,7 +23,13 @@ pub(crate) const FINGERPRINT_FILE: &str = "fingerprint.bin";
|
|||||||
/// The layer MPHF: `Xxh3Int` hasher, `CubicEps` buckets, multi-part (`SINGLE_PART = false`),
|
/// The layer MPHF: `Xxh3Int` hasher, `CubicEps` buckets, multi-part (`SINGLE_PART = false`),
|
||||||
/// minimal (`REMAP = true`). Built owned (construction + store); queried through
|
/// minimal (`REMAP = true`). Built owned (construction + store); queried through
|
||||||
/// `MemCase<Mphf>::uncase()`, the ε-copy view into the memory-mapped file.
|
/// `MemCase<Mphf>::uncase()`, the ε-copy view into the memory-mapped file.
|
||||||
pub(crate) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
pub type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||||
|
|
||||||
|
/// Construction parameters of [`Mphf`] — the one place they are chosen
|
||||||
|
/// (`lambda = 3.5`, `alpha = 0.99`, `CubicEps`, ~2.4 bits/key).
|
||||||
|
pub fn mphf_params() -> PtrHashParams<CubicEps> {
|
||||||
|
PtrHashParams::<CubicEps>::default_balanced()
|
||||||
|
}
|
||||||
|
|
||||||
// ── LayerEvidence ─────────────────────────────────────────────────────────────
|
// ── LayerEvidence ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -550,7 +556,7 @@ impl MphfLayer {
|
|||||||
|
|
||||||
// ── Empty layer ───────────────────────────────────────────────────────
|
// ── Empty layer ───────────────────────────────────────────────────────
|
||||||
if n == 0 {
|
if n == 0 {
|
||||||
let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::<CubicEps>::default_balanced())
|
let mphf: Mphf = Mphf::try_new(&[] as &[u64], mphf_params())
|
||||||
.ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?;
|
.ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?;
|
||||||
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
|
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
|
||||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||||
@@ -574,7 +580,7 @@ impl MphfLayer {
|
|||||||
let mphf: Mphf = Mphf::new_from_par_iter(
|
let mphf: Mphf = Mphf::new_from_par_iter(
|
||||||
n,
|
n,
|
||||||
keys.map(|k| k.raw()).par_bridge(),
|
keys.map(|k| k.raw()).par_bridge(),
|
||||||
PtrHashParams::<CubicEps>::default_balanced(),
|
mphf_params(),
|
||||||
);
|
);
|
||||||
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
|
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
|
||||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||||
|
|||||||
@@ -14,9 +14,7 @@ obicompactvec = { path = "../obicompactvec" }
|
|||||||
obipipeline = { path = "../obipipeline" }
|
obipipeline = { path = "../obipipeline" }
|
||||||
obiread = { path = "../obiread" }
|
obiread = { path = "../obiread" }
|
||||||
obiskbuilder = { path = "../obiskbuilder" }
|
obiskbuilder = { path = "../obiskbuilder" }
|
||||||
cacheline-ef = "2.0"
|
|
||||||
epserde = "0.13"
|
epserde = "0.13"
|
||||||
ptr_hash = { version = "2.1", features = ["epserde"] }
|
|
||||||
niffler = "3.0.0"
|
niffler = "3.0.0"
|
||||||
memmap2 = "0.9.10"
|
memmap2 = "0.9.10"
|
||||||
rayon = "1"
|
rayon = "1"
|
||||||
|
|||||||
@@ -3,18 +3,15 @@ use std::fs;
|
|||||||
use std::io;
|
use std::io;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use cacheline_ef::{CachelineEf, CachelineEfVec};
|
|
||||||
use epserde::ser::Serialize as EpSerialize;
|
use epserde::ser::Serialize as EpSerialize;
|
||||||
use memmap2::Mmap;
|
use memmap2::Mmap;
|
||||||
use obicompactvec::PersistentCompactIntVecBuilder;
|
use obicompactvec::PersistentCompactIntVecBuilder;
|
||||||
|
use obikindex::layer::{Mphf, mphf_params};
|
||||||
use obiskio::{SKFileReader, SKResult};
|
use obiskio::{SKFileReader, SKResult};
|
||||||
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int};
|
|
||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
|
|
||||||
use super::kmer_sort::sort_unique_kmers;
|
use super::kmer_sort::sort_unique_kmers;
|
||||||
|
|
||||||
pub(super) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
|
||||||
|
|
||||||
fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
|
fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
|
||||||
let file = fs::File::open(unique_path)?;
|
let file = fs::File::open(unique_path)?;
|
||||||
let mmap = unsafe { Mmap::map(&file)? };
|
let mmap = unsafe { Mmap::map(&file)? };
|
||||||
@@ -24,7 +21,7 @@ fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
|
|||||||
// Sequential constructor: the outer par_iter over partitions already saturates
|
// Sequential constructor: the outer par_iter over partitions already saturates
|
||||||
// the Rayon pool. new_from_par_iter would get no additional threads and adds
|
// the Rayon pool. new_from_par_iter would get no additional threads and adds
|
||||||
// coordination overhead. try_new accesses the same mmap'd pages at zero extra cost.
|
// coordination overhead. try_new accesses the same mmap'd pages at zero extra cost.
|
||||||
Mphf::try_new(kmers, PtrHashParams::<CubicEps>::default_balanced())
|
Mphf::try_new(kmers, mphf_params())
|
||||||
.ok_or_else(|| io::Error::other("ptr_hash construction failed"))
|
.ok_or_else(|| io::Error::other("ptr_hash construction failed"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -25,17 +25,13 @@ use std::io;
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
|
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
|
||||||
use cacheline_ef::{CachelineEf, CachelineEfVec};
|
|
||||||
use epserde::prelude::*;
|
use epserde::prelude::*;
|
||||||
use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec};
|
use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec};
|
||||||
use obidebruinj::GraphDeBruijn;
|
use obidebruinj::GraphDeBruijn;
|
||||||
use obikindex::layer::IndexMode;
|
use obikindex::layer::IndexMode;
|
||||||
use obikindex::layer::{KmerLayer, TypedLayer};
|
use obikindex::layer::{KmerLayer, Mphf, TypedLayer};
|
||||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||||
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
||||||
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xxh3Int};
|
|
||||||
|
|
||||||
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
|
||||||
|
|
||||||
pub(crate) trait PrivateBuilder {
|
pub(crate) trait PrivateBuilder {
|
||||||
/// Mark scatter as complete (`IndexState::Scattered`).
|
/// Mark scatter as complete (`IndexState::Scattered`).
|
||||||
|
|||||||
Generated
+3216
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,36 @@
|
|||||||
|
[package]
|
||||||
|
name = "migrate-mphf"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
publish = false
|
||||||
|
description = "One-shot converter: obikmer indexes whose layer MPHFs were built with ptr_hash 1.x -> ptr_hash 2.x"
|
||||||
|
|
||||||
|
# Standalone: not a member of src/'s workspace, own Cargo.lock. The previous
|
||||||
|
# generation of the MPHF stack (and its `proc-macro-error2` future-incompat
|
||||||
|
# warning) lives only here, never in obikmer itself.
|
||||||
|
[workspace]
|
||||||
|
|
||||||
|
[features]
|
||||||
|
default = []
|
||||||
|
numa = ["obisys/numa"]
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
obikindex = { path = "../../src/obikindex" }
|
||||||
|
obicompactvec = { path = "../../src/obicompactvec" }
|
||||||
|
obiskio = { path = "../../src/obiskio" }
|
||||||
|
obisys = { path = "../../src/obisys", default-features = false }
|
||||||
|
epserde = "0.13"
|
||||||
|
rayon = "1"
|
||||||
|
anyhow = "1"
|
||||||
|
clap = { version = "4", features = ["derive"] }
|
||||||
|
tracing = "0.1.44"
|
||||||
|
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
|
||||||
|
|
||||||
|
# The generation the indexes to convert were built with, under other names
|
||||||
|
# so that both can live in this one binary.
|
||||||
|
ptr_hash_old = { package = "ptr_hash", version = "=1.1.0" }
|
||||||
|
epserde_old = { package = "epserde", version = "=0.8.0" }
|
||||||
|
cacheline_ef_old = { package = "cacheline-ef", version = "=1.1.0" }
|
||||||
|
|
||||||
|
[profile.release]
|
||||||
|
debug = 1
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
//! Crash-safe replacement of a layer's files by their converted versions.
|
||||||
|
//!
|
||||||
|
//! Everything new is written under `<layer>/.migrate_mphf/` first; nothing of
|
||||||
|
//! the layer is touched until that is complete and verified. Then a `COMMIT`
|
||||||
|
//! marker is written, and [`finish`] moves the staged files into place. Every
|
||||||
|
//! step of [`finish`] is idempotent ("if the staged item still exists, move
|
||||||
|
//! it"), so a layer interrupted after `COMMIT` is simply finished by the next
|
||||||
|
//! run. Files are only ever replaced by `rename`, never written in place:
|
||||||
|
//! some of them are hard-linked from elsewhere (`ls -l` link count 2), and an
|
||||||
|
//! in-place write would corrupt the other link.
|
||||||
|
|
||||||
|
use std::fs;
|
||||||
|
use std::io;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
|
||||||
|
pub const STAGING: &str = ".migrate_mphf";
|
||||||
|
pub const COMMIT: &str = "COMMIT";
|
||||||
|
/// Written last: the layer is converted, skip it on later runs.
|
||||||
|
pub const DONE: &str = ".mphf_migrated";
|
||||||
|
/// Where derived annexes go instead of being deleted.
|
||||||
|
pub const STALE_DIR: &str = "stale_annexes_pre_ptrhash2";
|
||||||
|
|
||||||
|
/// Replaced as directories.
|
||||||
|
const DIRS: [&str; 2] = ["presence", "counts"];
|
||||||
|
/// Replaced as files; `mphf.bin` last.
|
||||||
|
const FILES: [&str; 3] = ["evidence.bin", "fingerprint.bin", "mphf.bin"];
|
||||||
|
|
||||||
|
/// Slot-ordered or slot-derived annexes of a layer: wrong (silently) once the
|
||||||
|
/// slot numbering changes, so they are set aside, to be rebuilt.
|
||||||
|
const LAYER_ANNEXES: [&str; 7] = [
|
||||||
|
"siblings.psib",
|
||||||
|
"entropy.pent",
|
||||||
|
"entropy.noamb.pent",
|
||||||
|
"pairweight.ppwt",
|
||||||
|
"pairweight.noamb.ppwt",
|
||||||
|
"quartet.pbiv",
|
||||||
|
"quartet.noamb.pbiv",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Index-level annexes derived from the sibling annex.
|
||||||
|
pub const INDEX_ANNEXES: [&str; 2] = ["family_overlap.povl", "family_overlap.noamb.povl"];
|
||||||
|
|
||||||
|
/// Move the staged files into `dir`, set the stale annexes aside, mark the
|
||||||
|
/// layer done and drop the staging area.
|
||||||
|
pub fn finish(dir: &Path) -> Result<()> {
|
||||||
|
let staging = dir.join(STAGING);
|
||||||
|
|
||||||
|
for name in DIRS {
|
||||||
|
let staged = staging.join(name);
|
||||||
|
if staged.exists() {
|
||||||
|
let target = dir.join(name);
|
||||||
|
if target.exists() {
|
||||||
|
fs::remove_dir_all(&target).with_context(|| format!("removing {}", target.display()))?;
|
||||||
|
}
|
||||||
|
fs::rename(&staged, &target).with_context(|| format!("installing {}", target.display()))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for name in FILES {
|
||||||
|
let staged = staging.join(name);
|
||||||
|
if staged.exists() {
|
||||||
|
let target = dir.join(name);
|
||||||
|
fs::rename(&staged, &target).with_context(|| format!("installing {}", target.display()))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
set_aside(dir, &LAYER_ANNEXES)?;
|
||||||
|
|
||||||
|
fs::write(dir.join(DONE), env!("CARGO_PKG_VERSION"))?;
|
||||||
|
fs::remove_dir_all(&staging)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Move each of `names` that exists in `dir` into `dir/STALE_DIR/`.
|
||||||
|
/// Returns how many were moved.
|
||||||
|
pub fn set_aside(dir: &Path, names: &[&str]) -> Result<usize> {
|
||||||
|
let aside = dir.join(STALE_DIR);
|
||||||
|
let mut moved = 0;
|
||||||
|
for name in names {
|
||||||
|
let from = dir.join(name);
|
||||||
|
if !from.exists() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
fs::create_dir_all(&aside)?;
|
||||||
|
fs::rename(&from, aside.join(name)).with_context(|| format!("setting aside {}", from.display()))?;
|
||||||
|
moved += 1;
|
||||||
|
}
|
||||||
|
Ok(moved)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove a staging area left by an interrupted, uncommitted run.
|
||||||
|
pub fn discard(dir: &Path) -> io::Result<()> {
|
||||||
|
match fs::remove_dir_all(dir.join(STAGING)) {
|
||||||
|
Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e),
|
||||||
|
_ => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
//! Permute the per-slot files of a layer into `staging/`.
|
||||||
|
//!
|
||||||
|
//! Presence and count matrices always come out in the compact row-major
|
||||||
|
//! (sparse) format, whatever they were: a dense source is first converted
|
||||||
|
//! sequentially (`build_from_dense`), then permuted from the sparse form,
|
||||||
|
//! whose rows are cheap to read at arbitrary slots.
|
||||||
|
|
||||||
|
use std::fs;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use anyhow::{Result, ensure};
|
||||||
|
use obicompactvec::{
|
||||||
|
PersistentBitMatrix, PersistentIntMatrix, PersistentSparseBitMatrixBuilder,
|
||||||
|
PersistentSparseCompactIntMatrixBuilder,
|
||||||
|
};
|
||||||
|
use obikindex::layer::evidence::{Evidence, EvidenceWriter};
|
||||||
|
use obikindex::layer::fingerprint::{FingerprintVec, FingerprintVecWriter};
|
||||||
|
|
||||||
|
use crate::plan::Plan;
|
||||||
|
use crate::util::oki;
|
||||||
|
|
||||||
|
/// Every per-slot file of the layer in `dir` that exists, permuted into `staging`.
|
||||||
|
pub fn permute_all(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
|
||||||
|
if dir.join("evidence.bin").exists() {
|
||||||
|
evidence(dir, staging, plan)?;
|
||||||
|
}
|
||||||
|
if dir.join("fingerprint.bin").exists() {
|
||||||
|
fingerprint(dir, staging, plan)?;
|
||||||
|
}
|
||||||
|
if dir.join("presence").is_dir() {
|
||||||
|
presence(dir, staging, plan)?;
|
||||||
|
}
|
||||||
|
if dir.join("counts").is_dir() {
|
||||||
|
counts(dir, staging, plan)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn evidence(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
|
||||||
|
let old = oki(Evidence::open(&dir.join("evidence.bin")))?;
|
||||||
|
ensure!(old.len() == plan.n, "evidence.bin has {} slots, layer has {}", old.len(), plan.n);
|
||||||
|
let mut out = EvidenceWriter::new(plan.n);
|
||||||
|
for (s, &o) in plan.new_to_old.iter().enumerate() {
|
||||||
|
let (chunk_id, rank) = old.decode(o as usize);
|
||||||
|
out.set(s, chunk_id, rank);
|
||||||
|
}
|
||||||
|
oki(out.write(&staging.join("evidence.bin")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fingerprint(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
|
||||||
|
let old = oki(FingerprintVec::open(&dir.join("fingerprint.bin")))?;
|
||||||
|
ensure!(old.n() == plan.n, "fingerprint.bin has {} slots, layer has {}", old.n(), plan.n);
|
||||||
|
let mut out = FingerprintVecWriter::new(plan.n, old.b());
|
||||||
|
for (s, &o) in plan.new_to_old.iter().enumerate() {
|
||||||
|
out.set(s, old.get(o as usize));
|
||||||
|
}
|
||||||
|
oki(out.write(&staging.join("fingerprint.bin")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn presence(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
|
||||||
|
let out = staging.join("presence");
|
||||||
|
let src = match PersistentBitMatrix::open(dir)? {
|
||||||
|
// Mono-genome: no file, nothing is attached to slots.
|
||||||
|
PersistentBitMatrix::Implicit { .. } => return Ok(()),
|
||||||
|
PersistentBitMatrix::Sparse(s) => {
|
||||||
|
ensure!(s.n() == plan.n, "presence has {} rows, layer has {}", s.n(), plan.n);
|
||||||
|
PersistentSparseBitMatrixBuilder::build_permuted(&s, &plan.new_to_old, &out)?.close()?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
dense => dense,
|
||||||
|
};
|
||||||
|
ensure!(src.n() == plan.n, "presence has {} rows, layer has {}", src.n(), plan.n);
|
||||||
|
let tmp = staging.join("presence_dense_to_sparse");
|
||||||
|
let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&src, &tmp)?.finish()?;
|
||||||
|
drop(src);
|
||||||
|
PersistentSparseBitMatrixBuilder::build_permuted(&sparse, &plan.new_to_old, &out)?.close()?;
|
||||||
|
drop(sparse);
|
||||||
|
fs::remove_dir_all(&tmp)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn counts(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
|
||||||
|
let out = staging.join("counts");
|
||||||
|
let src = match PersistentIntMatrix::open(dir)? {
|
||||||
|
PersistentIntMatrix::Sparse(s) => {
|
||||||
|
ensure!(s.n() == plan.n, "counts has {} rows, layer has {}", s.n(), plan.n);
|
||||||
|
PersistentSparseCompactIntMatrixBuilder::build_permuted(&s, &plan.new_to_old, &out)?.close()?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
dense => dense,
|
||||||
|
};
|
||||||
|
ensure!(src.n() == plan.n, "counts has {} rows, layer has {}", src.n(), plan.n);
|
||||||
|
let tmp = staging.join("counts_dense_to_sparse");
|
||||||
|
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&src, &tmp)?.finish()?;
|
||||||
|
drop(src);
|
||||||
|
PersistentSparseCompactIntMatrixBuilder::build_permuted(&sparse, &plan.new_to_old, &out)?.close()?;
|
||||||
|
drop(sparse);
|
||||||
|
fs::remove_dir_all(&tmp)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
//! Convert one layer directory: new MPHF, permuted per-slot data, verified,
|
||||||
|
//! then installed atomically (see `commit`).
|
||||||
|
|
||||||
|
use std::fs;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use anyhow::{Result, anyhow, ensure};
|
||||||
|
use epserde::prelude::*;
|
||||||
|
use obikindex::layer::{Mphf, mphf_params};
|
||||||
|
use obiskio::UnitigFileReader;
|
||||||
|
|
||||||
|
use crate::util::sk;
|
||||||
|
use crate::{commit, data, old, plan, verify};
|
||||||
|
|
||||||
|
pub struct Options {
|
||||||
|
/// How many k-mers per layer to compare old against new (ignored with `verify_full`).
|
||||||
|
pub verify_sample: usize,
|
||||||
|
pub verify_full: bool,
|
||||||
|
/// Do everything except installing the result.
|
||||||
|
pub dry_run: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum Outcome {
|
||||||
|
/// Already converted by an earlier run.
|
||||||
|
Skipped,
|
||||||
|
/// Interrupted after verification by an earlier run; installed now.
|
||||||
|
Resumed,
|
||||||
|
Converted { n: usize, verified: usize },
|
||||||
|
/// Everything but the installation.
|
||||||
|
Checked { n: usize, verified: usize },
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn convert(dir: &Path, opts: &Options) -> Result<Outcome> {
|
||||||
|
if dir.join(commit::DONE).exists() {
|
||||||
|
commit::discard(dir)?;
|
||||||
|
return Ok(Outcome::Skipped);
|
||||||
|
}
|
||||||
|
let staging = dir.join(commit::STAGING);
|
||||||
|
if staging.join(commit::COMMIT).exists() {
|
||||||
|
commit::finish(dir)?;
|
||||||
|
return Ok(Outcome::Resumed);
|
||||||
|
}
|
||||||
|
commit::discard(dir)?;
|
||||||
|
fs::create_dir_all(&staging)?;
|
||||||
|
|
||||||
|
let n = sk(UnitigFileReader::open_sequential(&dir.join("unitigs.bin")))?.n_kmers();
|
||||||
|
ensure!(n <= u32::MAX as usize, "layer has {n} k-mers, more than the 2^32 this tool handles");
|
||||||
|
|
||||||
|
let verified = if n == 0 {
|
||||||
|
empty_mphf(&staging)?;
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
let old_mphf = old::load(&dir.join("mphf.bin"))?;
|
||||||
|
let plan = plan::build(dir, &staging, n, &old_mphf)?;
|
||||||
|
data::permute_all(dir, &staging, &plan)?;
|
||||||
|
let stride = if opts.verify_full { 1 } else { (n / opts.verify_sample.max(1)).max(1) };
|
||||||
|
verify::check(dir, &staging, &plan, &old_mphf, stride)?
|
||||||
|
};
|
||||||
|
|
||||||
|
if opts.dry_run {
|
||||||
|
commit::discard(dir)?;
|
||||||
|
return Ok(Outcome::Checked { n, verified });
|
||||||
|
}
|
||||||
|
fs::write(staging.join(commit::COMMIT), b"")?;
|
||||||
|
commit::finish(dir)?;
|
||||||
|
Ok(Outcome::Converted { n, verified })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An empty layer still carries an MPHF (of nothing), like `MphfLayer::build` writes.
|
||||||
|
fn empty_mphf(staging: &Path) -> Result<()> {
|
||||||
|
let mphf = Mphf::try_new(&[] as &[u64], mphf_params())
|
||||||
|
.ok_or_else(|| anyhow!("empty MPHF construction failed"))?;
|
||||||
|
unsafe { mphf.store(staging.join("mphf.bin")) }.map_err(|e| anyhow!("storing the empty MPHF: {e}"))
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
//! One-shot converter for obikmer indexes whose layer MPHFs were built with
|
||||||
|
//! `ptr_hash` 1.x: rebuilds each MPHF with `ptr_hash` 2.x and carries the
|
||||||
|
//! per-slot data (evidence, fingerprint, presence/count matrices) over to the
|
||||||
|
//! new slot numbering, in place, layer by layer, verified before installing.
|
||||||
|
//!
|
||||||
|
//! Presence and count matrices come out in the compact row-major format.
|
||||||
|
//! Annexes derived from slot numbering (sibling, entropy, pair-weight,
|
||||||
|
//! quartet, family-overlap) are set aside under `stale_annexes_pre_ptrhash2/`
|
||||||
|
//! and must be rebuilt.
|
||||||
|
|
||||||
|
mod commit;
|
||||||
|
mod data;
|
||||||
|
mod layer;
|
||||||
|
mod old;
|
||||||
|
mod plan;
|
||||||
|
mod util;
|
||||||
|
mod verify;
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
use anyhow::{Result, ensure};
|
||||||
|
use clap::Parser;
|
||||||
|
use obikindex::KmerIndex;
|
||||||
|
use obisys::PartitionRunner;
|
||||||
|
use tracing::{info, warn};
|
||||||
|
|
||||||
|
use layer::{Options, Outcome};
|
||||||
|
use util::oki;
|
||||||
|
|
||||||
|
#[derive(Parser)]
|
||||||
|
#[command(about = "Convert an obikmer index from ptr_hash 1.x to ptr_hash 2.x MPHFs, in place")]
|
||||||
|
struct Cli {
|
||||||
|
/// Index directory (the one holding `index.meta` and `part_*`).
|
||||||
|
index: PathBuf,
|
||||||
|
|
||||||
|
/// Limit the number of CPUs used.
|
||||||
|
#[arg(long)]
|
||||||
|
cpu_max: Option<usize>,
|
||||||
|
|
||||||
|
/// K-mers compared old-vs-new per layer before installing it.
|
||||||
|
#[arg(long, default_value_t = 200_000)]
|
||||||
|
verify_sample: usize,
|
||||||
|
|
||||||
|
/// Compare every k-mer of every layer (slow).
|
||||||
|
#[arg(long)]
|
||||||
|
verify_full: bool,
|
||||||
|
|
||||||
|
/// Do everything except installing: convert into staging, verify, discard.
|
||||||
|
#[arg(long)]
|
||||||
|
dry_run: bool,
|
||||||
|
|
||||||
|
/// Only these partitions (repeatable). The index-level annexes are then left alone.
|
||||||
|
#[arg(long)]
|
||||||
|
partition: Vec<usize>,
|
||||||
|
|
||||||
|
/// Only these layers of each partition (repeatable). The index-level annexes are then left alone.
|
||||||
|
#[arg(long)]
|
||||||
|
layer: Vec<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() -> Result<()> {
|
||||||
|
tracing_subscriber::fmt()
|
||||||
|
.with_env_filter(tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()))
|
||||||
|
.init();
|
||||||
|
let cli = Cli::parse();
|
||||||
|
if let Some(n) = cli.cpu_max {
|
||||||
|
obisys::set_cpu_cap(n);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Holds the index's directory lock for the whole run, and sets k.
|
||||||
|
let index = oki(KmerIndex::open_lock(&cli.index))?;
|
||||||
|
let n_partitions = index.n_partitions();
|
||||||
|
let partial = !cli.partition.is_empty() || !cli.layer.is_empty();
|
||||||
|
let order: Vec<usize> = if cli.partition.is_empty() { (0..n_partitions).collect() } else { cli.partition.clone() };
|
||||||
|
ensure!(order.iter().all(|&p| p < n_partitions), "index has only {n_partitions} partitions");
|
||||||
|
|
||||||
|
let opts = Options { verify_sample: cli.verify_sample, verify_full: cli.verify_full, dry_run: cli.dry_run };
|
||||||
|
let converted = AtomicUsize::new(0);
|
||||||
|
let skipped = AtomicUsize::new(0);
|
||||||
|
|
||||||
|
let mut done = 0usize;
|
||||||
|
PartitionRunner::new().run(
|
||||||
|
&order,
|
||||||
|
|p| -> Result<Vec<(usize, Outcome)>> {
|
||||||
|
let n_layers = oki(index.n_layers(p))?;
|
||||||
|
let mut outcomes = Vec::new();
|
||||||
|
for l in (0..n_layers).filter(|l| cli.layer.is_empty() || cli.layer.contains(l)) {
|
||||||
|
let dir = index.layer_path(p, l);
|
||||||
|
let outcome = layer::convert(&dir, &opts)
|
||||||
|
.map_err(|e| e.context(format!("partition {p}, layer {l} ({})", dir.display())))?;
|
||||||
|
match outcome {
|
||||||
|
Outcome::Skipped => skipped.fetch_add(1, Ordering::Relaxed),
|
||||||
|
_ => converted.fetch_add(1, Ordering::Relaxed),
|
||||||
|
};
|
||||||
|
outcomes.push((l, outcome));
|
||||||
|
}
|
||||||
|
Ok(outcomes)
|
||||||
|
},
|
||||||
|
|p, outcomes, elapsed| {
|
||||||
|
done += 1;
|
||||||
|
info!("partition {p} ({done}/{}) in {:.1?}: {outcomes:?}", order.len(), elapsed);
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
|
||||||
|
if !cli.dry_run && !partial {
|
||||||
|
let moved = commit::set_aside(index.dir(), &commit::INDEX_ANNEXES)?;
|
||||||
|
if moved > 0 {
|
||||||
|
info!("{moved} index-level annex(es) set aside under {}", commit::STALE_DIR);
|
||||||
|
}
|
||||||
|
} else if !cli.dry_run {
|
||||||
|
warn!("partial run: index-level annexes (family overlap) not set aside; do it on the final pass");
|
||||||
|
}
|
||||||
|
info!(
|
||||||
|
"done: {} layer(s) {}, {} already converted",
|
||||||
|
converted.load(Ordering::Relaxed),
|
||||||
|
if cli.dry_run { "checked" } else { "converted" },
|
||||||
|
skipped.load(Ordering::Relaxed)
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
//! The previous MPHF generation (`ptr_hash` 1.x), kept only to read the
|
||||||
|
//! `mphf.bin` files of indexes built before the migration.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use anyhow::{Result, anyhow};
|
||||||
|
use cacheline_ef_old::{CachelineEf, CachelineEfVec};
|
||||||
|
use epserde_old::prelude::*;
|
||||||
|
use ptr_hash_old::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
||||||
|
|
||||||
|
/// Exactly the type `obikindex` used before the migration.
|
||||||
|
pub type OldMphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
|
||||||
|
|
||||||
|
pub fn load(path: &Path) -> Result<OldMphf> {
|
||||||
|
OldMphf::load_full(path)
|
||||||
|
.map_err(|e| anyhow!("{}: not a ptr_hash 1.x MPHF (already converted?): {e}", path.display()))
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
//! The slot permutation between a layer's old and new MPHF.
|
||||||
|
//!
|
||||||
|
//! A layer's MPHF gives each k-mer a slot, and every other per-slot file of
|
||||||
|
//! the layer (evidence, fingerprint, presence/count matrices) is indexed by
|
||||||
|
//! it. A new MPHF numbers the same k-mers differently, so the data has to
|
||||||
|
//! follow: the k-mers themselves come from `unitigs.bin`, which does not
|
||||||
|
//! depend on the MPHF at all.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use anyhow::{Result, anyhow, ensure};
|
||||||
|
use epserde::prelude::*;
|
||||||
|
use obikindex::layer::{Mphf, mphf_params};
|
||||||
|
use obiskio::CanonicalKmerIter;
|
||||||
|
use rayon::prelude::*;
|
||||||
|
|
||||||
|
use crate::old::OldMphf;
|
||||||
|
use crate::util::sk;
|
||||||
|
|
||||||
|
/// `new_to_old[s]` is the old slot of the k-mer that the new MPHF puts in
|
||||||
|
/// slot `s`. Always a bijection on `0..n` (checked while it is built).
|
||||||
|
pub struct Plan {
|
||||||
|
pub n: usize,
|
||||||
|
pub new_to_old: Vec<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the new MPHF of the layer whose `unitigs.bin` is in `dir`, store it
|
||||||
|
/// as `staging/mphf.bin`, and derive the permutation against `old`.
|
||||||
|
pub fn build(dir: &Path, staging: &Path, n: usize, old: &OldMphf) -> Result<Plan> {
|
||||||
|
let unitigs = dir.join("unitigs.bin");
|
||||||
|
|
||||||
|
// Same construction as `MphfLayer::build`: a clonable mmap iterator, bridged.
|
||||||
|
let keys = sk(CanonicalKmerIter::new(&unitigs))?;
|
||||||
|
let new: Mphf = Mphf::new_from_par_iter(n, keys.map(|k| k.raw()).par_bridge(), mphf_params());
|
||||||
|
unsafe { new.store(staging.join("mphf.bin")) }.map_err(|e| anyhow!("storing the new MPHF: {e}"))?;
|
||||||
|
|
||||||
|
let mut new_to_old = vec![u32::MAX; n];
|
||||||
|
let mut old_seen = vec![0u8; n.div_ceil(8)];
|
||||||
|
for kmer in sk(CanonicalKmerIter::new(&unitigs))? {
|
||||||
|
let key = kmer.raw();
|
||||||
|
let o = old.index(&key);
|
||||||
|
let s = new.index(&key);
|
||||||
|
ensure!(o < n, "old MPHF: slot {o} out of range (n = {n}) — is this layer's mphf.bin really the one of its unitigs.bin?");
|
||||||
|
ensure!(s < n, "new MPHF: slot {s} out of range (n = {n})");
|
||||||
|
ensure!(new_to_old[s] == u32::MAX, "new MPHF puts two k-mers in slot {s}");
|
||||||
|
let (byte, bit) = (o / 8, 1u8 << (o % 8));
|
||||||
|
ensure!(old_seen[byte] & bit == 0, "old MPHF puts two k-mers in slot {o}");
|
||||||
|
old_seen[byte] |= bit;
|
||||||
|
new_to_old[s] = o as u32;
|
||||||
|
}
|
||||||
|
// n distinct new slots, all in range, among n: the map is total.
|
||||||
|
Ok(Plan { n, new_to_old })
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
//! Error adapters: the libraries each have their own error type.
|
||||||
|
|
||||||
|
use anyhow::{Result, anyhow};
|
||||||
|
|
||||||
|
pub fn sk<T>(r: obiskio::SKResult<T>) -> Result<T> {
|
||||||
|
r.map_err(|e| anyhow!("{e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn oki<T>(r: obikindex::OKIResult<T>) -> Result<T> {
|
||||||
|
r.map_err(|e| anyhow!("{e}"))
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
//! Check the staged files against the layer as it still is, before anything
|
||||||
|
//! is replaced.
|
||||||
|
//!
|
||||||
|
//! For a sample of k-mers read from `unitigs.bin`, the old chain (old MPHF →
|
||||||
|
//! old slot → old data) must agree with the new one (staged MPHF, loaded the
|
||||||
|
//! way queries load it → new slot → staged data): same evidence, same
|
||||||
|
//! fingerprint, same presence set, same counts. A full check is the same loop
|
||||||
|
//! with every k-mer.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use anyhow::{Result, anyhow, ensure};
|
||||||
|
use epserde::prelude::*;
|
||||||
|
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseCompactIntMatrix};
|
||||||
|
use obikindex::layer::Mphf;
|
||||||
|
use obikindex::layer::evidence::Evidence;
|
||||||
|
use obikindex::layer::fingerprint::FingerprintVec;
|
||||||
|
use obiskio::CanonicalKmerIter;
|
||||||
|
|
||||||
|
use crate::old::OldMphf;
|
||||||
|
use crate::plan::Plan;
|
||||||
|
use crate::util::{oki, sk};
|
||||||
|
|
||||||
|
/// Both sides of one per-slot file, when the layer has it.
|
||||||
|
struct Pair<O, N> {
|
||||||
|
old: O,
|
||||||
|
new: N,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pair<O, N>(
|
||||||
|
exists: bool,
|
||||||
|
old: impl FnOnce() -> Result<O>,
|
||||||
|
new: impl FnOnce() -> Result<N>,
|
||||||
|
) -> Result<Option<Pair<O, N>>> {
|
||||||
|
Ok(if exists { Some(Pair { old: old()?, new: new()? }) } else { None })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns how many k-mers were compared.
|
||||||
|
pub fn check(dir: &Path, staging: &Path, plan: &Plan, old_mphf: &OldMphf, stride: usize) -> Result<usize> {
|
||||||
|
let staged_mphf = unsafe { Mphf::mmap(staging.join("mphf.bin"), Flags::empty()) }
|
||||||
|
.map_err(|e| anyhow!("staged mphf.bin does not load: {e}"))?;
|
||||||
|
let new_mphf = staged_mphf.uncase();
|
||||||
|
|
||||||
|
let evidence = pair(
|
||||||
|
staging.join("evidence.bin").exists(),
|
||||||
|
|| oki(Evidence::open(&dir.join("evidence.bin"))),
|
||||||
|
|| oki(Evidence::open(&staging.join("evidence.bin"))),
|
||||||
|
)?;
|
||||||
|
let fingerprint = pair(
|
||||||
|
staging.join("fingerprint.bin").exists(),
|
||||||
|
|| oki(FingerprintVec::open(&dir.join("fingerprint.bin"))),
|
||||||
|
|| oki(FingerprintVec::open(&staging.join("fingerprint.bin"))),
|
||||||
|
)?;
|
||||||
|
let presence = pair(
|
||||||
|
staging.join("presence").is_dir(),
|
||||||
|
|| Ok(PersistentBitMatrix::open(dir)?),
|
||||||
|
|| Ok(PersistentSparseBitMatrix::open(&staging.join("presence"))?),
|
||||||
|
)?;
|
||||||
|
let counts = pair(
|
||||||
|
staging.join("counts").is_dir(),
|
||||||
|
|| Ok(PersistentIntMatrix::open(dir)?),
|
||||||
|
|| Ok(PersistentSparseCompactIntMatrix::open(&staging.join("counts"))?),
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let mut compared = 0;
|
||||||
|
for (i, kmer) in sk(CanonicalKmerIter::new(&dir.join("unitigs.bin")))?.enumerate() {
|
||||||
|
if i % stride != 0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let key = kmer.raw();
|
||||||
|
let o = old_mphf.index(&key);
|
||||||
|
let s = new_mphf.index(&key);
|
||||||
|
ensure!(s < plan.n && plan.new_to_old[s] as usize == o, "k-mer #{i}: staged MPHF slot {s} disagrees with the plan (old slot {o})");
|
||||||
|
|
||||||
|
if let Some(p) = &evidence {
|
||||||
|
ensure!(p.old.decode(o) == p.new.decode(s), "k-mer #{i}: evidence differs (old slot {o}, new slot {s})");
|
||||||
|
}
|
||||||
|
if let Some(p) = &fingerprint {
|
||||||
|
ensure!(p.old.get(o) == p.new.get(s), "k-mer #{i}: fingerprint differs (old slot {o}, new slot {s})");
|
||||||
|
}
|
||||||
|
if let Some(p) = &presence {
|
||||||
|
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: presence row differs (old slot {o}, new slot {s})");
|
||||||
|
}
|
||||||
|
if let Some(p) = &counts {
|
||||||
|
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: count row differs (old slot {o}, new slot {s})");
|
||||||
|
}
|
||||||
|
compared += 1;
|
||||||
|
}
|
||||||
|
Ok(compared)
|
||||||
|
}
|
||||||
Reference in new issue
Block a user