Extract dereplication logic into obikderep crate and centralize paths

Move partition dereplication logic into a dedicated `obikderep` crate that implements a two-phase hash-split/merge strategy with Rayon for parallel processing. Centralize superkmer file path construction in `obilayeredmap` and update dependent crates to use the new pipeline and shared path helpers. Adjust test suites to explicitly invoke the dereplication step.
This commit is contained in:
Eric Coissac
2026-08-20 22:30:03 +02:00
parent 616cb76af3
commit abc51c2add
17 changed files with 534 additions and 97 deletions
+21
View File
@@ -1515,6 +1515,24 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "obikderep"
version = "0.1.0"
dependencies = [
"niffler",
"obikindex",
"obikrope",
"obikseq",
"obilayeredmap",
"obiskbuilder",
"obiskio",
"obisys",
"rayon",
"sysinfo",
"tempfile",
"tracing",
]
[[package]]
name = "obikentropy"
version = "0.1.0"
@@ -1565,6 +1583,7 @@ dependencies = [
"kodama",
"obidebruinj",
"obifastwrite",
"obikderep",
"obikindex",
"obikpartitionner",
"obikphylo",
@@ -1607,6 +1626,7 @@ dependencies = [
"memmap2",
"niffler",
"obicompactvec",
"obikderep",
"obikindex",
"obikrope",
"obikseq",
@@ -1632,6 +1652,7 @@ dependencies = [
"memmap2",
"ndarray",
"obicompactvec",
"obikderep",
"obikindex",
"obikpartitionner",
"obikseq",