Extract dereplication logic into obikderep crate and centralize paths

Move partition dereplication logic into a dedicated `obikderep` crate that implements a two-phase hash-split/merge strategy with Rayon for parallel processing. Centralize superkmer file path construction in `obilayeredmap` and update dependent crates to use the new pipeline and shared path helpers. Adjust test suites to explicitly invoke the dereplication step.
This commit is contained in:
Eric Coissac
2026-08-20 22:30:03 +02:00
parent 616cb76af3
commit abc51c2add
17 changed files with 534 additions and 97 deletions
+2 -2
View File
@@ -43,7 +43,7 @@ impl KmerIndex {
block_bits: u8,
) -> Result<usize, SKError> {
let layer0_dir = self.layer_dir(i, 0);
let dedup_path = layer0_dir.join("dereplicated.skmer.zst");
let dedup_path = obilayeredmap::dereplicated_superkmers_path(&layer0_dir);
if !dedup_path.exists() {
return Ok(0);
}
@@ -123,7 +123,7 @@ impl KmerIndex {
/// Deletes `dereplicated.skmer.zst` (+ sidecar), `mphf1.bin`, `counts1.bin`.
pub fn remove_build_artifacts(&self, i: usize) {
let layer0_dir = self.layer_dir(i, 0);
let dedup = layer0_dir.join("dereplicated.skmer.zst");
let dedup = obilayeredmap::dereplicated_superkmers_path(&layer0_dir);
remove_if_exists(&SKFileMeta::sidecar_path(&dedup));
remove_if_exists(&dedup);
remove_if_exists(&layer0_dir.join("mphf1.bin"));