Extract dereplication logic into obikderep crate and centralize paths
Move partition dereplication logic into a dedicated `obikderep` crate that implements a two-phase hash-split/merge strategy with Rayon for parallel processing. Centralize superkmer file path construction in `obilayeredmap` and update dependent crates to use the new pipeline and shared path helpers. Adjust test suites to explicitly invoke the dereplication step.
This commit is contained in:
@@ -43,7 +43,7 @@ impl KmerIndex {
|
||||
block_bits: u8,
|
||||
) -> Result<usize, SKError> {
|
||||
let layer0_dir = self.layer_dir(i, 0);
|
||||
let dedup_path = layer0_dir.join("dereplicated.skmer.zst");
|
||||
let dedup_path = obilayeredmap::dereplicated_superkmers_path(&layer0_dir);
|
||||
if !dedup_path.exists() {
|
||||
return Ok(0);
|
||||
}
|
||||
@@ -123,7 +123,7 @@ impl KmerIndex {
|
||||
/// Deletes `dereplicated.skmer.zst` (+ sidecar), `mphf1.bin`, `counts1.bin`.
|
||||
pub fn remove_build_artifacts(&self, i: usize) {
|
||||
let layer0_dir = self.layer_dir(i, 0);
|
||||
let dedup = layer0_dir.join("dereplicated.skmer.zst");
|
||||
let dedup = obilayeredmap::dereplicated_superkmers_path(&layer0_dir);
|
||||
remove_if_exists(&SKFileMeta::sidecar_path(&dedup));
|
||||
remove_if_exists(&dedup);
|
||||
remove_if_exists(&layer0_dir.join("mphf1.bin"));
|
||||
|
||||
Reference in New Issue
Block a user