Extract dereplication logic into obikderep crate and centralize paths

Move partition dereplication logic into a dedicated `obikderep` crate that implements a two-phase hash-split/merge strategy with Rayon for parallel processing. Centralize superkmer file path construction in `obilayeredmap` and update dependent crates to use the new pipeline and shared path helpers. Adjust test suites to explicitly invoke the dereplication step.
This commit is contained in:
Eric Coissac committed 2026-08-20 22:30:03 +02:00
1 parent 616cb76af3
commit abc51c2add
17 files changed
+534 -97

No files matched your search

+21
View File
@@ -1515,6 +1515,24 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "obikderep"
version = "0.1.0"
dependencies = [
"niffler",
"obikindex",
"obikrope",
"obikseq",
"obilayeredmap",
"obiskbuilder",
"obiskio",
"obisys",
"rayon",
"sysinfo",
"tempfile",
"tracing",
]
[[package]]
name = "obikentropy"
version = "0.1.0"
@@ -1565,6 +1583,7 @@ dependencies = [
"kodama",
"obidebruinj",
"obifastwrite",
"obikderep",
"obikindex",
"obikpartitionner",
"obikphylo",
@@ -1607,6 +1626,7 @@ dependencies = [
"memmap2",
"niffler",
"obicompactvec",
"obikderep",
"obikindex",
"obikrope",
"obikseq",
@@ -1632,6 +1652,7 @@ dependencies = [
"memmap2",
"ndarray",
"obicompactvec",
"obikderep",
"obikindex",
"obikpartitionner",
"obikseq",