Implement MPHF migration tool and sparse matrix permutation logic
Refactors sparse matrix handling to implement row permutation logic and introduces a standalone tool for migrating indexes from MPHF 1.x to 2.x formats.
This commit is contained in:
1 parent
d686d39506
commit
8b87078478
24 files changed
+3982
-19
No files matched your search
@@ -62,7 +62,8 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
src/target
|
||||
key: linux-musl-cargo-${{ hashFiles('src/Cargo.lock') }}
|
||||
tools/migrate-mphf/target
|
||||
key: linux-musl-cargo-${{ hashFiles('src/Cargo.lock', 'tools/migrate-mphf/Cargo.lock') }}
|
||||
restore-keys: linux-musl-cargo-
|
||||
|
||||
- name: Build static binary
|
||||
@@ -83,6 +84,28 @@ jobs:
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-F "attachment=@/tmp/dist/obikmer-linux-x86_64"
|
||||
|
||||
# One-shot converter for indexes built with ptr_hash 1.x. A standalone
|
||||
# crate (own Cargo.lock/target, outside src/'s workspace), built after
|
||||
# obikmer's own asset is already uploaded: it never blocks that one.
|
||||
- name: Build migrate-mphf (static)
|
||||
working-directory: tools/migrate-mphf
|
||||
env:
|
||||
PKG_CONFIG_ALLOW_CROSS: "1"
|
||||
run: cargo zigbuild --release --target x86_64-unknown-linux-musl
|
||||
|
||||
- name: Upload migrate-mphf artifact
|
||||
working-directory: tools/migrate-mphf
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEATOKEN }}
|
||||
RELEASE_ID: ${{ needs.create-release.outputs.release_id }}
|
||||
run: |
|
||||
cp target/x86_64-unknown-linux-musl/release/migrate-mphf /tmp/dist/migrate-mphf-linux-x86_64
|
||||
strip /tmp/dist/migrate-mphf-linux-x86_64
|
||||
curl -s -X POST \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases/$RELEASE_ID/assets" \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-F "attachment=@/tmp/dist/migrate-mphf-linux-x86_64"
|
||||
|
||||
build-macos-arm64:
|
||||
needs: create-release
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -9,6 +9,7 @@ CLAUDE.md
|
||||
memory/
|
||||
sandbox/
|
||||
src/target
|
||||
tools/migrate-mphf/target
|
||||
data-stress
|
||||
*.fasta
|
||||
*.fasta.gz
|
||||
|
||||
@@ -42,6 +42,18 @@ obikmer:
|
||||
debug:
|
||||
cargo build --manifest-path $(CARGO_DIR)/Cargo.toml --bin obikmer
|
||||
|
||||
# One-shot converter of indexes built with ptr_hash 1.x (see tools/migrate-mphf).
|
||||
# Standalone crate, outside the src/ workspace, with its own Cargo.lock/target.
|
||||
MIGRATE_MPHF_DIR := tools/migrate-mphf
|
||||
|
||||
.PHONY: migrate-mphf
|
||||
migrate-mphf:
|
||||
cargo build --release --manifest-path $(MIGRATE_MPHF_DIR)/Cargo.toml
|
||||
|
||||
.PHONY: clean-migrate-mphf
|
||||
clean-migrate-mphf:
|
||||
rm -rf $(MIGRATE_MPHF_DIR)/target
|
||||
|
||||
.PHONY: profile-build
|
||||
profile-build:
|
||||
cargo build --release --manifest-path $(CARGO_DIR)/Cargo.toml \
|
||||
|
||||
Generated
-2
@@ -1528,7 +1528,6 @@ dependencies = [
|
||||
name = "obikindexer"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"cacheline-ef",
|
||||
"epserde",
|
||||
"memmap2",
|
||||
"niffler",
|
||||
@@ -1543,7 +1542,6 @@ dependencies = [
|
||||
"obiskbuilder",
|
||||
"obiskio",
|
||||
"obisys",
|
||||
"ptr_hash",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
||||
@@ -514,6 +514,31 @@ impl PersistentSparseBitMatrixBuilder {
|
||||
}
|
||||
Ok(builder)
|
||||
}
|
||||
|
||||
/// Builds the row-permuted copy of `src` into `dir`: row `s` of the
|
||||
/// result is row `new_to_old[s]` of `src`. `new_to_old` must have
|
||||
/// `src.n()` entries and be a permutation of `0..src.n()` — the caller
|
||||
/// proves that, this only reads `src` at those slots.
|
||||
///
|
||||
/// Needed whenever the slot numbering of a layer changes (its MPHF is
|
||||
/// rebuilt) while the data attached to each k-mer must follow it.
|
||||
/// Rows are read at arbitrary slots of `src`, so convert a dense source
|
||||
/// to sparse first (`build_from_dense`) rather than permuting it
|
||||
/// cell by cell.
|
||||
pub fn build_permuted(
|
||||
src: &PersistentSparseBitMatrix,
|
||||
new_to_old: &[u32],
|
||||
dir: &Path,
|
||||
) -> io::Result<Self> {
|
||||
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
|
||||
let mut genomes: Vec<u32> = Vec::new();
|
||||
for &old in new_to_old {
|
||||
genomes.clear();
|
||||
src.for_each_genome_in_row(old as usize, |g| genomes.push(g as u32));
|
||||
builder.push_row(&genomes);
|
||||
}
|
||||
Ok(builder)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Trait impls ───────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -548,6 +548,29 @@ impl PersistentSparseCompactIntMatrixBuilder {
|
||||
Ok(builder)
|
||||
}
|
||||
|
||||
/// Row-permuted copy of `src`: row `s` of the result is row
|
||||
/// `new_to_old[s]` of `src`. Same contract as
|
||||
/// [`PersistentSparseBitMatrixBuilder::build_permuted`].
|
||||
pub fn build_permuted(
|
||||
src: &PersistentSparseCompactIntMatrix,
|
||||
new_to_old: &[u32],
|
||||
dir: &Path,
|
||||
) -> io::Result<Self> {
|
||||
let mut builder = Self::new(new_to_old.len(), src.n_cols(), dir)?;
|
||||
let mut cols: Vec<u32> = Vec::new();
|
||||
let mut vals: Vec<u32> = Vec::new();
|
||||
for &old in new_to_old {
|
||||
cols.clear();
|
||||
vals.clear();
|
||||
src.for_each_cell_in_row(old as usize, |c, v| {
|
||||
cols.push(c as u32);
|
||||
vals.push(v);
|
||||
});
|
||||
builder.push_row(&cols, &vals);
|
||||
}
|
||||
Ok(builder)
|
||||
}
|
||||
|
||||
pub fn close(self) -> io::Result<()> {
|
||||
let Self {
|
||||
support,
|
||||
|
||||
@@ -415,3 +415,31 @@ fn partial_jaccard_and_hamming_match_dense_including_diagonal() {
|
||||
drop(dir_dense);
|
||||
drop(dir_sparse);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_permuted_follows_the_permutation() {
|
||||
// Mix of singleton and multi rows, including a repeated multi set, so the
|
||||
// dictionary dedup is exercised on the permuted order too.
|
||||
let rows: Vec<&[bool]> = vec![
|
||||
&[true, false, false, false],
|
||||
&[false, false, true, false],
|
||||
&[true, true, false, false],
|
||||
&[false, false, true, true],
|
||||
&[true, true, false, false],
|
||||
];
|
||||
let (_dir, src) = make_sparse(&rows, 4);
|
||||
let new_to_old: Vec<u32> = vec![3, 0, 4, 1, 2];
|
||||
|
||||
let out = tempdir().unwrap();
|
||||
let permuted =
|
||||
PersistentSparseBitMatrixBuilder::build_permuted(&src, &new_to_old, &out.path().join("p"))
|
||||
.unwrap()
|
||||
.finish()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(permuted.n(), src.n());
|
||||
assert_eq!(permuted.n_cols(), src.n_cols());
|
||||
for (new, &old) in new_to_old.iter().enumerate() {
|
||||
assert_eq!(row_as_bool(&permuted, new), row_as_bool(&src, old as usize), "new row {new}");
|
||||
}
|
||||
}
|
||||
@@ -304,3 +304,32 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() {
|
||||
assert_eq!(&*sparse.row(slot), &**expected, "slot {slot}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_permuted_follows_the_permutation() {
|
||||
// Rows sharing a support with different values, plus singletons.
|
||||
let rows: Vec<&[u32]> = vec![
|
||||
&[7, 0, 0, 0],
|
||||
&[0, 0, 3, 0],
|
||||
&[2, 5, 0, 0],
|
||||
&[9, 1, 0, 0],
|
||||
&[0, 0, 4, 6],
|
||||
];
|
||||
let (_dir, src) = make_sparse(&rows, 4);
|
||||
let new_to_old: Vec<u32> = vec![4, 2, 0, 3, 1];
|
||||
|
||||
let out = tempdir().unwrap();
|
||||
let permuted = PersistentSparseCompactIntMatrixBuilder::build_permuted(
|
||||
&src,
|
||||
&new_to_old,
|
||||
&out.path().join("p"),
|
||||
)
|
||||
.unwrap()
|
||||
.finish()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(permuted.n(), src.n());
|
||||
for (new, &old) in new_to_old.iter().enumerate() {
|
||||
assert_eq!(permuted.row(new), src.row(old as usize), "new row {new}");
|
||||
}
|
||||
}
|
||||
@@ -210,6 +210,14 @@ impl KmerIndex {
|
||||
crate::partition::index_dir(&self.root_path, i)
|
||||
}
|
||||
|
||||
/// Path of layer `j` of partition `i` — pure path computation, nothing
|
||||
/// is opened (unlike [`layer`](Self::layer)/`layer_dir`, which open the
|
||||
/// layer's MPHF). For maintenance tools that must reach a layer's files
|
||||
/// without going through a format they can't read yet.
|
||||
pub fn layer_path(&self, i: usize, j: usize) -> PathBuf {
|
||||
crate::layer::utils::layer_dir(&self.index_dir(i), j)
|
||||
}
|
||||
|
||||
/// Number of layers in partition `i` — a disk scan, not a metadata read
|
||||
/// (see [`KmerPartition::n_layers`](crate::partition::KmerPartition::n_layers)).
|
||||
pub fn n_layers(&self, i: usize) -> OKIResult<usize> {
|
||||
|
||||
@@ -9,5 +9,7 @@ pub(crate) mod utils;
|
||||
pub use crate::index::error::{OKIError, OKIResult};
|
||||
pub use content_layer::KmerLayer;
|
||||
pub use meta::IndexMode;
|
||||
pub use mphf_layer::{EvidenceKind, KmerBatchIter, KmerIter, MphfLayer, MphfOnly, UnitigIter};
|
||||
pub use mphf_layer::{
|
||||
EvidenceKind, KmerBatchIter, KmerIter, Mphf, MphfLayer, MphfOnly, UnitigIter, mphf_params,
|
||||
};
|
||||
pub use typed_layer::{HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer};
|
||||
@@ -23,7 +23,13 @@ pub(crate) const FINGERPRINT_FILE: &str = "fingerprint.bin";
|
||||
/// The layer MPHF: `Xxh3Int` hasher, `CubicEps` buckets, multi-part (`SINGLE_PART = false`),
|
||||
/// minimal (`REMAP = true`). Built owned (construction + store); queried through
|
||||
/// `MemCase<Mphf>::uncase()`, the ε-copy view into the memory-mapped file.
|
||||
pub(crate) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||
pub type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||
|
||||
/// Construction parameters of [`Mphf`] — the one place they are chosen
|
||||
/// (`lambda = 3.5`, `alpha = 0.99`, `CubicEps`, ~2.4 bits/key).
|
||||
pub fn mphf_params() -> PtrHashParams<CubicEps> {
|
||||
PtrHashParams::<CubicEps>::default_balanced()
|
||||
}
|
||||
|
||||
// ── LayerEvidence ─────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -550,7 +556,7 @@ impl MphfLayer {
|
||||
|
||||
// ── Empty layer ───────────────────────────────────────────────────────
|
||||
if n == 0 {
|
||||
let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::<CubicEps>::default_balanced())
|
||||
let mphf: Mphf = Mphf::try_new(&[] as &[u64], mphf_params())
|
||||
.ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?;
|
||||
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
|
||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||
@@ -574,7 +580,7 @@ impl MphfLayer {
|
||||
let mphf: Mphf = Mphf::new_from_par_iter(
|
||||
n,
|
||||
keys.map(|k| k.raw()).par_bridge(),
|
||||
PtrHashParams::<CubicEps>::default_balanced(),
|
||||
mphf_params(),
|
||||
);
|
||||
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
|
||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||
|
||||
@@ -14,9 +14,7 @@ obicompactvec = { path = "../obicompactvec" }
|
||||
obipipeline = { path = "../obipipeline" }
|
||||
obiread = { path = "../obiread" }
|
||||
obiskbuilder = { path = "../obiskbuilder" }
|
||||
cacheline-ef = "2.0"
|
||||
epserde = "0.13"
|
||||
ptr_hash = { version = "2.1", features = ["epserde"] }
|
||||
niffler = "3.0.0"
|
||||
memmap2 = "0.9.10"
|
||||
rayon = "1"
|
||||
|
||||
@@ -3,18 +3,15 @@ use std::fs;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use cacheline_ef::{CachelineEf, CachelineEfVec};
|
||||
use epserde::ser::Serialize as EpSerialize;
|
||||
use memmap2::Mmap;
|
||||
use obicompactvec::PersistentCompactIntVecBuilder;
|
||||
use obikindex::layer::{Mphf, mphf_params};
|
||||
use obiskio::{SKFileReader, SKResult};
|
||||
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int};
|
||||
use tracing::debug;
|
||||
|
||||
use super::kmer_sort::sort_unique_kmers;
|
||||
|
||||
pub(super) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||
|
||||
fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
|
||||
let file = fs::File::open(unique_path)?;
|
||||
let mmap = unsafe { Mmap::map(&file)? };
|
||||
@@ -24,7 +21,7 @@ fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
|
||||
// Sequential constructor: the outer par_iter over partitions already saturates
|
||||
// the Rayon pool. new_from_par_iter would get no additional threads and adds
|
||||
// coordination overhead. try_new accesses the same mmap'd pages at zero extra cost.
|
||||
Mphf::try_new(kmers, PtrHashParams::<CubicEps>::default_balanced())
|
||||
Mphf::try_new(kmers, mphf_params())
|
||||
.ok_or_else(|| io::Error::other("ptr_hash construction failed"))
|
||||
}
|
||||
|
||||
|
||||
@@ -25,17 +25,13 @@ use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
|
||||
use cacheline_ef::{CachelineEf, CachelineEfVec};
|
||||
use epserde::prelude::*;
|
||||
use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec};
|
||||
use obidebruinj::GraphDeBruijn;
|
||||
use obikindex::layer::IndexMode;
|
||||
use obikindex::layer::{KmerLayer, TypedLayer};
|
||||
use obikindex::layer::{KmerLayer, Mphf, TypedLayer};
|
||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
||||
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xxh3Int};
|
||||
|
||||
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||
|
||||
pub(crate) trait PrivateBuilder {
|
||||
/// Mark scatter as complete (`IndexState::Scattered`).
|
||||
|
||||
Generated
+3216
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,36 @@
|
||||
[package]
|
||||
name = "migrate-mphf"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
publish = false
|
||||
description = "One-shot converter: obikmer indexes whose layer MPHFs were built with ptr_hash 1.x -> ptr_hash 2.x"
|
||||
|
||||
# Standalone: not a member of src/'s workspace, own Cargo.lock. The previous
|
||||
# generation of the MPHF stack (and its `proc-macro-error2` future-incompat
|
||||
# warning) lives only here, never in obikmer itself.
|
||||
[workspace]
|
||||
|
||||
[features]
|
||||
default = []
|
||||
numa = ["obisys/numa"]
|
||||
|
||||
[dependencies]
|
||||
obikindex = { path = "../../src/obikindex" }
|
||||
obicompactvec = { path = "../../src/obicompactvec" }
|
||||
obiskio = { path = "../../src/obiskio" }
|
||||
obisys = { path = "../../src/obisys", default-features = false }
|
||||
epserde = "0.13"
|
||||
rayon = "1"
|
||||
anyhow = "1"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
tracing = "0.1.44"
|
||||
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
|
||||
|
||||
# The generation the indexes to convert were built with, under other names
|
||||
# so that both can live in this one binary.
|
||||
ptr_hash_old = { package = "ptr_hash", version = "=1.1.0" }
|
||||
epserde_old = { package = "epserde", version = "=0.8.0" }
|
||||
cacheline_ef_old = { package = "cacheline-ef", version = "=1.1.0" }
|
||||
|
||||
[profile.release]
|
||||
debug = 1
|
||||
@@ -0,0 +1,98 @@
|
||||
//! Crash-safe replacement of a layer's files by their converted versions.
|
||||
//!
|
||||
//! Everything new is written under `<layer>/.migrate_mphf/` first; nothing of
|
||||
//! the layer is touched until that is complete and verified. Then a `COMMIT`
|
||||
//! marker is written, and [`finish`] moves the staged files into place. Every
|
||||
//! step of [`finish`] is idempotent ("if the staged item still exists, move
|
||||
//! it"), so a layer interrupted after `COMMIT` is simply finished by the next
|
||||
//! run. Files are only ever replaced by `rename`, never written in place:
|
||||
//! some of them are hard-linked from elsewhere (`ls -l` link count 2), and an
|
||||
//! in-place write would corrupt the other link.
|
||||
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
pub const STAGING: &str = ".migrate_mphf";
|
||||
pub const COMMIT: &str = "COMMIT";
|
||||
/// Written last: the layer is converted, skip it on later runs.
|
||||
pub const DONE: &str = ".mphf_migrated";
|
||||
/// Where derived annexes go instead of being deleted.
|
||||
pub const STALE_DIR: &str = "stale_annexes_pre_ptrhash2";
|
||||
|
||||
/// Replaced as directories.
|
||||
const DIRS: [&str; 2] = ["presence", "counts"];
|
||||
/// Replaced as files; `mphf.bin` last.
|
||||
const FILES: [&str; 3] = ["evidence.bin", "fingerprint.bin", "mphf.bin"];
|
||||
|
||||
/// Slot-ordered or slot-derived annexes of a layer: wrong (silently) once the
|
||||
/// slot numbering changes, so they are set aside, to be rebuilt.
|
||||
const LAYER_ANNEXES: [&str; 7] = [
|
||||
"siblings.psib",
|
||||
"entropy.pent",
|
||||
"entropy.noamb.pent",
|
||||
"pairweight.ppwt",
|
||||
"pairweight.noamb.ppwt",
|
||||
"quartet.pbiv",
|
||||
"quartet.noamb.pbiv",
|
||||
];
|
||||
|
||||
/// Index-level annexes derived from the sibling annex.
|
||||
pub const INDEX_ANNEXES: [&str; 2] = ["family_overlap.povl", "family_overlap.noamb.povl"];
|
||||
|
||||
/// Move the staged files into `dir`, set the stale annexes aside, mark the
|
||||
/// layer done and drop the staging area.
|
||||
pub fn finish(dir: &Path) -> Result<()> {
|
||||
let staging = dir.join(STAGING);
|
||||
|
||||
for name in DIRS {
|
||||
let staged = staging.join(name);
|
||||
if staged.exists() {
|
||||
let target = dir.join(name);
|
||||
if target.exists() {
|
||||
fs::remove_dir_all(&target).with_context(|| format!("removing {}", target.display()))?;
|
||||
}
|
||||
fs::rename(&staged, &target).with_context(|| format!("installing {}", target.display()))?;
|
||||
}
|
||||
}
|
||||
for name in FILES {
|
||||
let staged = staging.join(name);
|
||||
if staged.exists() {
|
||||
let target = dir.join(name);
|
||||
fs::rename(&staged, &target).with_context(|| format!("installing {}", target.display()))?;
|
||||
}
|
||||
}
|
||||
|
||||
set_aside(dir, &LAYER_ANNEXES)?;
|
||||
|
||||
fs::write(dir.join(DONE), env!("CARGO_PKG_VERSION"))?;
|
||||
fs::remove_dir_all(&staging)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Move each of `names` that exists in `dir` into `dir/STALE_DIR/`.
|
||||
/// Returns how many were moved.
|
||||
pub fn set_aside(dir: &Path, names: &[&str]) -> Result<usize> {
|
||||
let aside = dir.join(STALE_DIR);
|
||||
let mut moved = 0;
|
||||
for name in names {
|
||||
let from = dir.join(name);
|
||||
if !from.exists() {
|
||||
continue;
|
||||
}
|
||||
fs::create_dir_all(&aside)?;
|
||||
fs::rename(&from, aside.join(name)).with_context(|| format!("setting aside {}", from.display()))?;
|
||||
moved += 1;
|
||||
}
|
||||
Ok(moved)
|
||||
}
|
||||
|
||||
/// Remove a staging area left by an interrupted, uncommitted run.
|
||||
pub fn discard(dir: &Path) -> io::Result<()> {
|
||||
match fs::remove_dir_all(dir.join(STAGING)) {
|
||||
Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
//! Permute the per-slot files of a layer into `staging/`.
|
||||
//!
|
||||
//! Presence and count matrices always come out in the compact row-major
|
||||
//! (sparse) format, whatever they were: a dense source is first converted
|
||||
//! sequentially (`build_from_dense`), then permuted from the sparse form,
|
||||
//! whose rows are cheap to read at arbitrary slots.
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Result, ensure};
|
||||
use obicompactvec::{
|
||||
PersistentBitMatrix, PersistentIntMatrix, PersistentSparseBitMatrixBuilder,
|
||||
PersistentSparseCompactIntMatrixBuilder,
|
||||
};
|
||||
use obikindex::layer::evidence::{Evidence, EvidenceWriter};
|
||||
use obikindex::layer::fingerprint::{FingerprintVec, FingerprintVecWriter};
|
||||
|
||||
use crate::plan::Plan;
|
||||
use crate::util::oki;
|
||||
|
||||
/// Every per-slot file of the layer in `dir` that exists, permuted into `staging`.
|
||||
pub fn permute_all(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
|
||||
if dir.join("evidence.bin").exists() {
|
||||
evidence(dir, staging, plan)?;
|
||||
}
|
||||
if dir.join("fingerprint.bin").exists() {
|
||||
fingerprint(dir, staging, plan)?;
|
||||
}
|
||||
if dir.join("presence").is_dir() {
|
||||
presence(dir, staging, plan)?;
|
||||
}
|
||||
if dir.join("counts").is_dir() {
|
||||
counts(dir, staging, plan)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn evidence(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
|
||||
let old = oki(Evidence::open(&dir.join("evidence.bin")))?;
|
||||
ensure!(old.len() == plan.n, "evidence.bin has {} slots, layer has {}", old.len(), plan.n);
|
||||
let mut out = EvidenceWriter::new(plan.n);
|
||||
for (s, &o) in plan.new_to_old.iter().enumerate() {
|
||||
let (chunk_id, rank) = old.decode(o as usize);
|
||||
out.set(s, chunk_id, rank);
|
||||
}
|
||||
oki(out.write(&staging.join("evidence.bin")))
|
||||
}
|
||||
|
||||
fn fingerprint(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
|
||||
let old = oki(FingerprintVec::open(&dir.join("fingerprint.bin")))?;
|
||||
ensure!(old.n() == plan.n, "fingerprint.bin has {} slots, layer has {}", old.n(), plan.n);
|
||||
let mut out = FingerprintVecWriter::new(plan.n, old.b());
|
||||
for (s, &o) in plan.new_to_old.iter().enumerate() {
|
||||
out.set(s, old.get(o as usize));
|
||||
}
|
||||
oki(out.write(&staging.join("fingerprint.bin")))
|
||||
}
|
||||
|
||||
fn presence(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
|
||||
let out = staging.join("presence");
|
||||
let src = match PersistentBitMatrix::open(dir)? {
|
||||
// Mono-genome: no file, nothing is attached to slots.
|
||||
PersistentBitMatrix::Implicit { .. } => return Ok(()),
|
||||
PersistentBitMatrix::Sparse(s) => {
|
||||
ensure!(s.n() == plan.n, "presence has {} rows, layer has {}", s.n(), plan.n);
|
||||
PersistentSparseBitMatrixBuilder::build_permuted(&s, &plan.new_to_old, &out)?.close()?;
|
||||
return Ok(());
|
||||
}
|
||||
dense => dense,
|
||||
};
|
||||
ensure!(src.n() == plan.n, "presence has {} rows, layer has {}", src.n(), plan.n);
|
||||
let tmp = staging.join("presence_dense_to_sparse");
|
||||
let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&src, &tmp)?.finish()?;
|
||||
drop(src);
|
||||
PersistentSparseBitMatrixBuilder::build_permuted(&sparse, &plan.new_to_old, &out)?.close()?;
|
||||
drop(sparse);
|
||||
fs::remove_dir_all(&tmp)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn counts(dir: &Path, staging: &Path, plan: &Plan) -> Result<()> {
|
||||
let out = staging.join("counts");
|
||||
let src = match PersistentIntMatrix::open(dir)? {
|
||||
PersistentIntMatrix::Sparse(s) => {
|
||||
ensure!(s.n() == plan.n, "counts has {} rows, layer has {}", s.n(), plan.n);
|
||||
PersistentSparseCompactIntMatrixBuilder::build_permuted(&s, &plan.new_to_old, &out)?.close()?;
|
||||
return Ok(());
|
||||
}
|
||||
dense => dense,
|
||||
};
|
||||
ensure!(src.n() == plan.n, "counts has {} rows, layer has {}", src.n(), plan.n);
|
||||
let tmp = staging.join("counts_dense_to_sparse");
|
||||
let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&src, &tmp)?.finish()?;
|
||||
drop(src);
|
||||
PersistentSparseCompactIntMatrixBuilder::build_permuted(&sparse, &plan.new_to_old, &out)?.close()?;
|
||||
drop(sparse);
|
||||
fs::remove_dir_all(&tmp)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
//! Convert one layer directory: new MPHF, permuted per-slot data, verified,
|
||||
//! then installed atomically (see `commit`).
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Result, anyhow, ensure};
|
||||
use epserde::prelude::*;
|
||||
use obikindex::layer::{Mphf, mphf_params};
|
||||
use obiskio::UnitigFileReader;
|
||||
|
||||
use crate::util::sk;
|
||||
use crate::{commit, data, old, plan, verify};
|
||||
|
||||
pub struct Options {
|
||||
/// How many k-mers per layer to compare old against new (ignored with `verify_full`).
|
||||
pub verify_sample: usize,
|
||||
pub verify_full: bool,
|
||||
/// Do everything except installing the result.
|
||||
pub dry_run: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Outcome {
|
||||
/// Already converted by an earlier run.
|
||||
Skipped,
|
||||
/// Interrupted after verification by an earlier run; installed now.
|
||||
Resumed,
|
||||
Converted { n: usize, verified: usize },
|
||||
/// Everything but the installation.
|
||||
Checked { n: usize, verified: usize },
|
||||
}
|
||||
|
||||
pub fn convert(dir: &Path, opts: &Options) -> Result<Outcome> {
|
||||
if dir.join(commit::DONE).exists() {
|
||||
commit::discard(dir)?;
|
||||
return Ok(Outcome::Skipped);
|
||||
}
|
||||
let staging = dir.join(commit::STAGING);
|
||||
if staging.join(commit::COMMIT).exists() {
|
||||
commit::finish(dir)?;
|
||||
return Ok(Outcome::Resumed);
|
||||
}
|
||||
commit::discard(dir)?;
|
||||
fs::create_dir_all(&staging)?;
|
||||
|
||||
let n = sk(UnitigFileReader::open_sequential(&dir.join("unitigs.bin")))?.n_kmers();
|
||||
ensure!(n <= u32::MAX as usize, "layer has {n} k-mers, more than the 2^32 this tool handles");
|
||||
|
||||
let verified = if n == 0 {
|
||||
empty_mphf(&staging)?;
|
||||
0
|
||||
} else {
|
||||
let old_mphf = old::load(&dir.join("mphf.bin"))?;
|
||||
let plan = plan::build(dir, &staging, n, &old_mphf)?;
|
||||
data::permute_all(dir, &staging, &plan)?;
|
||||
let stride = if opts.verify_full { 1 } else { (n / opts.verify_sample.max(1)).max(1) };
|
||||
verify::check(dir, &staging, &plan, &old_mphf, stride)?
|
||||
};
|
||||
|
||||
if opts.dry_run {
|
||||
commit::discard(dir)?;
|
||||
return Ok(Outcome::Checked { n, verified });
|
||||
}
|
||||
fs::write(staging.join(commit::COMMIT), b"")?;
|
||||
commit::finish(dir)?;
|
||||
Ok(Outcome::Converted { n, verified })
|
||||
}
|
||||
|
||||
/// An empty layer still carries an MPHF (of nothing), like `MphfLayer::build` writes.
|
||||
fn empty_mphf(staging: &Path) -> Result<()> {
|
||||
let mphf = Mphf::try_new(&[] as &[u64], mphf_params())
|
||||
.ok_or_else(|| anyhow!("empty MPHF construction failed"))?;
|
||||
unsafe { mphf.store(staging.join("mphf.bin")) }.map_err(|e| anyhow!("storing the empty MPHF: {e}"))
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
//! One-shot converter for obikmer indexes whose layer MPHFs were built with
|
||||
//! `ptr_hash` 1.x: rebuilds each MPHF with `ptr_hash` 2.x and carries the
|
||||
//! per-slot data (evidence, fingerprint, presence/count matrices) over to the
|
||||
//! new slot numbering, in place, layer by layer, verified before installing.
|
||||
//!
|
||||
//! Presence and count matrices come out in the compact row-major format.
|
||||
//! Annexes derived from slot numbering (sibling, entropy, pair-weight,
|
||||
//! quartet, family-overlap) are set aside under `stale_annexes_pre_ptrhash2/`
|
||||
//! and must be rebuilt.
|
||||
|
||||
mod commit;
|
||||
mod data;
|
||||
mod layer;
|
||||
mod old;
|
||||
mod plan;
|
||||
mod util;
|
||||
mod verify;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use anyhow::{Result, ensure};
|
||||
use clap::Parser;
|
||||
use obikindex::KmerIndex;
|
||||
use obisys::PartitionRunner;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use layer::{Options, Outcome};
|
||||
use util::oki;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(about = "Convert an obikmer index from ptr_hash 1.x to ptr_hash 2.x MPHFs, in place")]
|
||||
struct Cli {
|
||||
/// Index directory (the one holding `index.meta` and `part_*`).
|
||||
index: PathBuf,
|
||||
|
||||
/// Limit the number of CPUs used.
|
||||
#[arg(long)]
|
||||
cpu_max: Option<usize>,
|
||||
|
||||
/// K-mers compared old-vs-new per layer before installing it.
|
||||
#[arg(long, default_value_t = 200_000)]
|
||||
verify_sample: usize,
|
||||
|
||||
/// Compare every k-mer of every layer (slow).
|
||||
#[arg(long)]
|
||||
verify_full: bool,
|
||||
|
||||
/// Do everything except installing: convert into staging, verify, discard.
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
|
||||
/// Only these partitions (repeatable). The index-level annexes are then left alone.
|
||||
#[arg(long)]
|
||||
partition: Vec<usize>,
|
||||
|
||||
/// Only these layers of each partition (repeatable). The index-level annexes are then left alone.
|
||||
#[arg(long)]
|
||||
layer: Vec<usize>,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()))
|
||||
.init();
|
||||
let cli = Cli::parse();
|
||||
if let Some(n) = cli.cpu_max {
|
||||
obisys::set_cpu_cap(n);
|
||||
}
|
||||
|
||||
// Holds the index's directory lock for the whole run, and sets k.
|
||||
let index = oki(KmerIndex::open_lock(&cli.index))?;
|
||||
let n_partitions = index.n_partitions();
|
||||
let partial = !cli.partition.is_empty() || !cli.layer.is_empty();
|
||||
let order: Vec<usize> = if cli.partition.is_empty() { (0..n_partitions).collect() } else { cli.partition.clone() };
|
||||
ensure!(order.iter().all(|&p| p < n_partitions), "index has only {n_partitions} partitions");
|
||||
|
||||
let opts = Options { verify_sample: cli.verify_sample, verify_full: cli.verify_full, dry_run: cli.dry_run };
|
||||
let converted = AtomicUsize::new(0);
|
||||
let skipped = AtomicUsize::new(0);
|
||||
|
||||
let mut done = 0usize;
|
||||
PartitionRunner::new().run(
|
||||
&order,
|
||||
|p| -> Result<Vec<(usize, Outcome)>> {
|
||||
let n_layers = oki(index.n_layers(p))?;
|
||||
let mut outcomes = Vec::new();
|
||||
for l in (0..n_layers).filter(|l| cli.layer.is_empty() || cli.layer.contains(l)) {
|
||||
let dir = index.layer_path(p, l);
|
||||
let outcome = layer::convert(&dir, &opts)
|
||||
.map_err(|e| e.context(format!("partition {p}, layer {l} ({})", dir.display())))?;
|
||||
match outcome {
|
||||
Outcome::Skipped => skipped.fetch_add(1, Ordering::Relaxed),
|
||||
_ => converted.fetch_add(1, Ordering::Relaxed),
|
||||
};
|
||||
outcomes.push((l, outcome));
|
||||
}
|
||||
Ok(outcomes)
|
||||
},
|
||||
|p, outcomes, elapsed| {
|
||||
done += 1;
|
||||
info!("partition {p} ({done}/{}) in {:.1?}: {outcomes:?}", order.len(), elapsed);
|
||||
},
|
||||
)?;
|
||||
|
||||
if !cli.dry_run && !partial {
|
||||
let moved = commit::set_aside(index.dir(), &commit::INDEX_ANNEXES)?;
|
||||
if moved > 0 {
|
||||
info!("{moved} index-level annex(es) set aside under {}", commit::STALE_DIR);
|
||||
}
|
||||
} else if !cli.dry_run {
|
||||
warn!("partial run: index-level annexes (family overlap) not set aside; do it on the final pass");
|
||||
}
|
||||
info!(
|
||||
"done: {} layer(s) {}, {} already converted",
|
||||
converted.load(Ordering::Relaxed),
|
||||
if cli.dry_run { "checked" } else { "converted" },
|
||||
skipped.load(Ordering::Relaxed)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
//! The previous MPHF generation (`ptr_hash` 1.x), kept only to read the
|
||||
//! `mphf.bin` files of indexes built before the migration.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Result, anyhow};
|
||||
use cacheline_ef_old::{CachelineEf, CachelineEfVec};
|
||||
use epserde_old::prelude::*;
|
||||
use ptr_hash_old::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
||||
|
||||
/// Exactly the type `obikindex` used before the migration.
|
||||
pub type OldMphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
|
||||
|
||||
pub fn load(path: &Path) -> Result<OldMphf> {
|
||||
OldMphf::load_full(path)
|
||||
.map_err(|e| anyhow!("{}: not a ptr_hash 1.x MPHF (already converted?): {e}", path.display()))
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
//! The slot permutation between a layer's old and new MPHF.
|
||||
//!
|
||||
//! A layer's MPHF gives each k-mer a slot, and every other per-slot file of
|
||||
//! the layer (evidence, fingerprint, presence/count matrices) is indexed by
|
||||
//! it. A new MPHF numbers the same k-mers differently, so the data has to
|
||||
//! follow: the k-mers themselves come from `unitigs.bin`, which does not
|
||||
//! depend on the MPHF at all.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Result, anyhow, ensure};
|
||||
use epserde::prelude::*;
|
||||
use obikindex::layer::{Mphf, mphf_params};
|
||||
use obiskio::CanonicalKmerIter;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::old::OldMphf;
|
||||
use crate::util::sk;
|
||||
|
||||
/// `new_to_old[s]` is the old slot of the k-mer that the new MPHF puts in
|
||||
/// slot `s`. Always a bijection on `0..n` (checked while it is built).
|
||||
pub struct Plan {
|
||||
pub n: usize,
|
||||
pub new_to_old: Vec<u32>,
|
||||
}
|
||||
|
||||
/// Build the new MPHF of the layer whose `unitigs.bin` is in `dir`, store it
|
||||
/// as `staging/mphf.bin`, and derive the permutation against `old`.
|
||||
pub fn build(dir: &Path, staging: &Path, n: usize, old: &OldMphf) -> Result<Plan> {
|
||||
let unitigs = dir.join("unitigs.bin");
|
||||
|
||||
// Same construction as `MphfLayer::build`: a clonable mmap iterator, bridged.
|
||||
let keys = sk(CanonicalKmerIter::new(&unitigs))?;
|
||||
let new: Mphf = Mphf::new_from_par_iter(n, keys.map(|k| k.raw()).par_bridge(), mphf_params());
|
||||
unsafe { new.store(staging.join("mphf.bin")) }.map_err(|e| anyhow!("storing the new MPHF: {e}"))?;
|
||||
|
||||
let mut new_to_old = vec![u32::MAX; n];
|
||||
let mut old_seen = vec![0u8; n.div_ceil(8)];
|
||||
for kmer in sk(CanonicalKmerIter::new(&unitigs))? {
|
||||
let key = kmer.raw();
|
||||
let o = old.index(&key);
|
||||
let s = new.index(&key);
|
||||
ensure!(o < n, "old MPHF: slot {o} out of range (n = {n}) — is this layer's mphf.bin really the one of its unitigs.bin?");
|
||||
ensure!(s < n, "new MPHF: slot {s} out of range (n = {n})");
|
||||
ensure!(new_to_old[s] == u32::MAX, "new MPHF puts two k-mers in slot {s}");
|
||||
let (byte, bit) = (o / 8, 1u8 << (o % 8));
|
||||
ensure!(old_seen[byte] & bit == 0, "old MPHF puts two k-mers in slot {o}");
|
||||
old_seen[byte] |= bit;
|
||||
new_to_old[s] = o as u32;
|
||||
}
|
||||
// n distinct new slots, all in range, among n: the map is total.
|
||||
Ok(Plan { n, new_to_old })
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
//! Error adapters: the libraries each have their own error type.
|
||||
|
||||
use anyhow::{Result, anyhow};
|
||||
|
||||
pub fn sk<T>(r: obiskio::SKResult<T>) -> Result<T> {
|
||||
r.map_err(|e| anyhow!("{e}"))
|
||||
}
|
||||
|
||||
pub fn oki<T>(r: obikindex::OKIResult<T>) -> Result<T> {
|
||||
r.map_err(|e| anyhow!("{e}"))
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
//! Check the staged files against the layer as it still is, before anything
|
||||
//! is replaced.
|
||||
//!
|
||||
//! For a sample of k-mers read from `unitigs.bin`, the old chain (old MPHF →
|
||||
//! old slot → old data) must agree with the new one (staged MPHF, loaded the
|
||||
//! way queries load it → new slot → staged data): same evidence, same
|
||||
//! fingerprint, same presence set, same counts. A full check is the same loop
|
||||
//! with every k-mer.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Result, anyhow, ensure};
|
||||
use epserde::prelude::*;
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseCompactIntMatrix};
|
||||
use obikindex::layer::Mphf;
|
||||
use obikindex::layer::evidence::Evidence;
|
||||
use obikindex::layer::fingerprint::FingerprintVec;
|
||||
use obiskio::CanonicalKmerIter;
|
||||
|
||||
use crate::old::OldMphf;
|
||||
use crate::plan::Plan;
|
||||
use crate::util::{oki, sk};
|
||||
|
||||
/// Both sides of one per-slot file, when the layer has it.
|
||||
struct Pair<O, N> {
|
||||
old: O,
|
||||
new: N,
|
||||
}
|
||||
|
||||
fn pair<O, N>(
|
||||
exists: bool,
|
||||
old: impl FnOnce() -> Result<O>,
|
||||
new: impl FnOnce() -> Result<N>,
|
||||
) -> Result<Option<Pair<O, N>>> {
|
||||
Ok(if exists { Some(Pair { old: old()?, new: new()? }) } else { None })
|
||||
}
|
||||
|
||||
/// Returns how many k-mers were compared.
|
||||
pub fn check(dir: &Path, staging: &Path, plan: &Plan, old_mphf: &OldMphf, stride: usize) -> Result<usize> {
|
||||
let staged_mphf = unsafe { Mphf::mmap(staging.join("mphf.bin"), Flags::empty()) }
|
||||
.map_err(|e| anyhow!("staged mphf.bin does not load: {e}"))?;
|
||||
let new_mphf = staged_mphf.uncase();
|
||||
|
||||
let evidence = pair(
|
||||
staging.join("evidence.bin").exists(),
|
||||
|| oki(Evidence::open(&dir.join("evidence.bin"))),
|
||||
|| oki(Evidence::open(&staging.join("evidence.bin"))),
|
||||
)?;
|
||||
let fingerprint = pair(
|
||||
staging.join("fingerprint.bin").exists(),
|
||||
|| oki(FingerprintVec::open(&dir.join("fingerprint.bin"))),
|
||||
|| oki(FingerprintVec::open(&staging.join("fingerprint.bin"))),
|
||||
)?;
|
||||
let presence = pair(
|
||||
staging.join("presence").is_dir(),
|
||||
|| Ok(PersistentBitMatrix::open(dir)?),
|
||||
|| Ok(PersistentSparseBitMatrix::open(&staging.join("presence"))?),
|
||||
)?;
|
||||
let counts = pair(
|
||||
staging.join("counts").is_dir(),
|
||||
|| Ok(PersistentIntMatrix::open(dir)?),
|
||||
|| Ok(PersistentSparseCompactIntMatrix::open(&staging.join("counts"))?),
|
||||
)?;
|
||||
|
||||
let mut compared = 0;
|
||||
for (i, kmer) in sk(CanonicalKmerIter::new(&dir.join("unitigs.bin")))?.enumerate() {
|
||||
if i % stride != 0 {
|
||||
continue;
|
||||
}
|
||||
let key = kmer.raw();
|
||||
let o = old_mphf.index(&key);
|
||||
let s = new_mphf.index(&key);
|
||||
ensure!(s < plan.n && plan.new_to_old[s] as usize == o, "k-mer #{i}: staged MPHF slot {s} disagrees with the plan (old slot {o})");
|
||||
|
||||
if let Some(p) = &evidence {
|
||||
ensure!(p.old.decode(o) == p.new.decode(s), "k-mer #{i}: evidence differs (old slot {o}, new slot {s})");
|
||||
}
|
||||
if let Some(p) = &fingerprint {
|
||||
ensure!(p.old.get(o) == p.new.get(s), "k-mer #{i}: fingerprint differs (old slot {o}, new slot {s})");
|
||||
}
|
||||
if let Some(p) = &presence {
|
||||
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: presence row differs (old slot {o}, new slot {s})");
|
||||
}
|
||||
if let Some(p) = &counts {
|
||||
ensure!(p.old.row(o) == p.new.row(s), "k-mer #{i}: count row differs (old slot {o}, new slot {s})");
|
||||
}
|
||||
compared += 1;
|
||||
}
|
||||
Ok(compared)
|
||||
}
|
||||
Reference in new issue
Block a user