Refactor indexing logic and update core dependencies

This commit introduces significant updates to the indexing system, including the introduction of zero-copy types, modifications to the `Mphf` type definition, and new hybrid search strategies. Core algorithms are updated to handle new hashing types and improved error handling during pointer hash construction. Dependency versions are also updated across the board, including new security features and system-level crates.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:07:08 +02:00
1 parent 3bd7fd4f41
commit d686d39506
6 files changed
+185 -205

No files matched your search

+151 -169
View File
@@ -30,6 +30,12 @@ dependencies = [
"memchr",
]
[[package]]
name = "aliasable"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd"
[[package]]
name = "allocator-api2"
version = "0.2.21"
@@ -167,12 +173,6 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.11.1"
@@ -191,6 +191,15 @@ dependencies = [
"wyz",
]
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]]
name = "block-buffer"
version = "0.12.1"
@@ -268,11 +277,11 @@ dependencies = [
[[package]]
name = "cacheline-ef"
version = "1.1.0"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af737c6c59cb018ecbe6472cbdf86d39c59d78252febfe311953a991b6e4ed85"
checksum = "6e701af8ea9f58c4ba18793a5d5dffd791cda99bf766976c59c583939d180fd5"
dependencies = [
"common_traits 0.11.4",
"common_traits",
"epserde",
"mem_dbg",
]
@@ -312,6 +321,12 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "cfg_aliases"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
[[package]]
name = "chacha20"
version = "0.10.1"
@@ -319,7 +334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
dependencies = [
"cfg-if",
"cpufeatures",
"cpufeatures 0.3.0",
"rand_core 0.10.1",
]
@@ -433,17 +448,6 @@ dependencies = [
"memchr",
]
[[package]]
name = "common_traits"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fda9ae1f26adcae83adb2e92f69cf59421f2a277a942f49f8e59f2fcbd7cf062"
dependencies = [
"anyhow",
"half",
"impl-tools 0.10.3",
]
[[package]]
name = "common_traits"
version = "0.13.0"
@@ -452,7 +456,7 @@ checksum = "d8e0d7d0c5977b1e87f759d406740c38227125f374f54d81bc197e877f2c5f26"
dependencies = [
"anyhow",
"half",
"impl-tools 0.12.0",
"impl-tools",
]
[[package]]
@@ -482,6 +486,15 @@ dependencies = [
"unicode-segmentation",
]
[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
"libc",
]
[[package]]
name = "cpufeatures"
version = "0.3.0"
@@ -592,6 +605,16 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-common"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "crypto-common"
version = "0.2.2"
@@ -645,15 +668,25 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"crypto-common 0.1.7",
]
[[package]]
name = "digest"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
dependencies = [
"block-buffer",
"block-buffer 0.12.1",
"const-oid",
"crypto-common",
"crypto-common 0.2.2",
]
[[package]]
@@ -662,7 +695,7 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
dependencies = [
"bitflags 2.11.1",
"bitflags",
"objc2",
]
@@ -698,27 +731,27 @@ dependencies = [
[[package]]
name = "epserde"
version = "0.8.0"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c40d342ff20a2ce62d9a85ce406e672dfa137f902ac9670034533184f1533976"
checksum = "d6cd9f3212ce43e7b594a974c6e8cef4e11c6473d76b7b97cb01c239130789eb"
dependencies = [
"aliasable",
"anyhow",
"bitflags 2.11.1",
"common_traits 0.11.4",
"bitflags",
"epserde-derive",
"maligned",
"maybe-dangling",
"mem_dbg",
"mmap-rs",
"sealed",
"sha2",
"thiserror 2.0.18",
"xxhash-rust",
]
[[package]]
name = "epserde-derive"
version = "0.8.0"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac80cc78b69765703f48ad93f33b8919cf5d907cda7459ad6ba2919cbbe605dd"
checksum = "d73e3083270b2ff144c990e26580374fdfd512f61dd5a12da9b5bd749a7585e0"
dependencies = [
"proc-macro2",
"quote",
@@ -806,6 +839,16 @@ dependencies = [
"byteorder",
]
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.17"
@@ -852,6 +895,17 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
[[package]]
name = "hashbrown"
version = "0.17.1"
@@ -899,7 +953,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c2e65a48d3b300843ac84a2fe8e166bb5a5b00f30054593bcee8157e4b465fd"
dependencies = [
"arrayvec",
"bitflags 2.11.1",
"bitflags",
"derive_more",
"errno",
"hwlocality-sys",
@@ -935,18 +989,6 @@ dependencies = [
"typenum",
]
[[package]]
name = "impl-tools"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ae95c9095c2f1126d7db785955c73cdc5fc33e7c3fa911bd4a42931672029a7"
dependencies = [
"autocfg",
"impl-tools-lib 0.11.4",
"proc-macro-error2",
"syn 2.0.117",
]
[[package]]
name = "impl-tools"
version = "0.12.0"
@@ -954,23 +996,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06c8567c75b167cdda5ecee024da9cc9581d41bd52460e2e0e768967972aeef1"
dependencies = [
"autocfg",
"impl-tools-lib 0.12.0",
"impl-tools-lib",
"proc-macro-error3",
"syn 2.0.117",
]
[[package]]
name = "impl-tools-lib"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab699036df31c1f7d3561bfa6e9cb9bc3bb0fd2e2cd9bf121c31cb961d049ddf"
dependencies = [
"proc-macro-error2",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "impl-tools-lib"
version = "0.12.0"
@@ -990,7 +1020,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
dependencies = [
"equivalent",
"hashbrown",
"hashbrown 0.17.1",
]
[[package]]
@@ -1043,9 +1073,9 @@ dependencies = [
[[package]]
name = "itertools"
version = "0.14.0"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
dependencies = [
"either",
]
@@ -1083,7 +1113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa"
dependencies = [
"cfg-if",
"cpufeatures",
"cpufeatures 0.3.0",
]
[[package]]
@@ -1157,7 +1187,7 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a"
dependencies = [
"hashbrown",
"hashbrown 0.17.1",
]
[[package]]
@@ -1169,12 +1199,6 @@ dependencies = [
"libc",
]
[[package]]
name = "maligned"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e88c3cbe8288f77f293e48a28b3232e3defd203a6d839fa7f68ea4329e83464"
[[package]]
name = "matchers"
version = "0.2.0"
@@ -1195,22 +1219,30 @@ dependencies = [
]
[[package]]
name = "mem_dbg"
version = "0.3.4"
name = "maybe-dangling"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "728cc9dc97593cd22f7bc81fbef70a2d391d7a9a855e7d658b653318124a6cf0"
checksum = "59dbb09ed53f8e4f314e353dc6c1853ae5b4c480a668a422657804a544ea9f65"
[[package]]
name = "mem_dbg"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b48a1086c746f4ee6ca5cb0acf856a14709bc4d2d20e03db150a12ddf2269e6d"
dependencies = [
"bitflags 2.11.1",
"maligned",
"aliasable",
"bitflags",
"hashbrown 0.16.1",
"maybe-dangling",
"mem_dbg-derive",
"mmap-rs",
]
[[package]]
name = "mem_dbg-derive"
version = "0.2.1"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d84f40c93b0508d5565db79a814d02d5b2545967205ce44be211592aafa34d6c"
checksum = "eb910efe8da52f13da727170e352e50a1764579a6fb1065d00d9556da19c79ac"
dependencies = [
"proc-macro2",
"quote",
@@ -1232,15 +1264,6 @@ dependencies = [
"libc",
]
[[package]]
name = "memoffset"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
dependencies = [
"autocfg",
]
[[package]]
name = "miniz_oxide"
version = "0.8.9"
@@ -1253,17 +1276,17 @@ dependencies = [
[[package]]
name = "mmap-rs"
version = "0.6.1"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86968d85441db75203c34deefd0c88032f275aaa85cee19a1dcfff6ae9df56da"
checksum = "4ecce9d566cb9234ae3db9e249c8b55665feaaf32b0859ff1e27e310d2beb3d8"
dependencies = [
"bitflags 1.3.2",
"bitflags",
"combine",
"libc",
"mach2",
"nix",
"sysctl",
"thiserror 1.0.69",
"thiserror 2.0.18",
"widestring",
"windows 0.48.0",
]
@@ -1326,15 +1349,14 @@ dependencies = [
[[package]]
name = "nix"
version = "0.26.4"
version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
dependencies = [
"bitflags 1.3.2",
"bitflags",
"cfg-if",
"cfg_aliases",
"libc",
"memoffset",
"pin-utils",
]
[[package]]
@@ -1386,7 +1408,7 @@ dependencies = [
name = "obicompactvec"
version = "0.1.0"
dependencies = [
"common_traits 0.13.0",
"common_traits",
"memmap2",
"ndarray",
"rayon",
@@ -1399,7 +1421,7 @@ version = "0.1.0"
dependencies = [
"ahash",
"crossbeam-channel",
"hashbrown",
"hashbrown 0.17.1",
"obifastwrite",
"obikseq",
"rayon",
@@ -1773,7 +1795,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
dependencies = [
"bitflags 2.11.1",
"bitflags",
"dispatch2",
"objc2",
]
@@ -1790,7 +1812,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
dependencies = [
"bitflags 2.11.1",
"bitflags",
"objc2",
]
@@ -1884,12 +1906,6 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
[[package]]
name = "pin-utils"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.33"
@@ -1949,14 +1965,10 @@ dependencies = [
]
[[package]]
name = "proc-macro-error-attr2"
version = "2.0.0"
name = "prefetch-index"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5"
dependencies = [
"proc-macro2",
"quote",
]
checksum = "9057806a8d77d67bccdc0f542db43737a6f19ada3efab2adc63277feea27310f"
[[package]]
name = "proc-macro-error-attr3"
@@ -1968,17 +1980,6 @@ dependencies = [
"quote",
]
[[package]]
name = "proc-macro-error2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
dependencies = [
"proc-macro-error-attr2",
"proc-macro2",
"quote",
]
[[package]]
name = "proc-macro-error3"
version = "3.1.0"
@@ -2001,30 +2002,25 @@ dependencies = [
[[package]]
name = "ptr_hash"
version = "1.1.0"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b4e4fb9c4c2ba3e5b060f53ef46afd3de37345b08e3ec0f2c65e0ca1d57ccbd"
checksum = "b22cf1be774892680e056d3387db0dcaa4fd46964b270671ef661e127186b969"
dependencies = [
"anyhow",
"bitvec",
"cacheline-ef",
"clap",
"colored",
"common_traits 0.11.4",
"epserde",
"epserde-derive",
"fastrand",
"fxhash",
"itertools 0.14.0",
"lazy_static",
"itertools 0.15.0",
"log",
"mem_dbg",
"rand 0.9.4",
"rand_chacha 0.9.0",
"prefetch-index",
"rand 0.10.2",
"rand_chacha 0.10.0",
"rayon",
"rdst",
"rustc-hash",
"serde",
"sucds",
"tempfile",
"xxhash-rust",
@@ -2097,16 +2093,6 @@ dependencies = [
"rand_core 0.6.4",
]
[[package]]
name = "rand"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
dependencies = [
"rand_chacha 0.9.0",
"rand_core 0.9.5",
]
[[package]]
name = "rand"
version = "0.10.2"
@@ -2130,12 +2116,12 @@ dependencies = [
[[package]]
name = "rand_chacha"
version = "0.9.0"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb"
dependencies = [
"ppv-lite86",
"rand_core 0.9.5",
"rand_core 0.10.1",
]
[[package]]
@@ -2147,15 +2133,6 @@ dependencies = [
"getrandom 0.2.17",
]
[[package]]
name = "rand_core"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_core"
version = "0.10.1"
@@ -2215,7 +2192,7 @@ version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags 2.11.1",
"bitflags",
]
[[package]]
@@ -2278,7 +2255,7 @@ checksum = "d9776093b7ca170454ab1406954f7b7d97a57c51dc6c0642957fb2ef25c2d399"
dependencies = [
"bytecheck",
"bytes",
"hashbrown",
"hashbrown 0.17.1",
"indexmap",
"munge",
"ptr_meta",
@@ -2300,12 +2277,6 @@ dependencies = [
"syn 3.0.3",
]
[[package]]
name = "rustc-hash"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
[[package]]
name = "rustc_version"
version = "0.4.1"
@@ -2321,7 +2292,7 @@ version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
dependencies = [
"bitflags 2.11.1",
"bitflags",
"errno",
"libc",
"linux-raw-sys",
@@ -2463,13 +2434,24 @@ dependencies = [
"unsafe-libyaml",
]
[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures 0.2.17",
"digest 0.10.7",
]
[[package]]
name = "sha3"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1"
dependencies = [
"digest",
"digest 0.11.3",
"keccak",
]
@@ -2592,11 +2574,11 @@ dependencies = [
[[package]]
name = "sysctl"
version = "0.5.5"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"
checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc"
dependencies = [
"bitflags 2.11.1",
"bitflags",
"byteorder",
"enum-as-inner",
"libc",
+3 -3
View File
@@ -12,9 +12,9 @@ obicompactvec = { path = "../obicompactvec" }
obidebruinj = { path = "../obidebruinj" }
obipipeline = { path = "../obipipeline" }
obikentropy = { path = "../obikentropy" }
cacheline-ef = "1.1"
epserde = "0.8"
ptr_hash = "1.1"
cacheline-ef = "2.0"
epserde = "0.13"
ptr_hash = { version = "2.1", features = ["epserde"] }
niffler = "3.0.0"
memmap2 = "0.9.10"
ndarray = "0.17"
+21 -23
View File
@@ -8,7 +8,7 @@ use epserde::prelude::*;
use obicompactvec::LayerMeta;
use obikseq::CanonicalKmer;
use obiskio::{CanonicalKmerIter, UnitigFileReader, UnitigFileWriter, build_unitig_idx};
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64};
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int};
use crate::index::error::{OKIError, OKIResult};
use crate::layer::evidence::{Evidence, EvidenceWriter};
@@ -20,12 +20,10 @@ pub(crate) const UNITIGS_FILE: &str = "unitigs.bin";
pub(crate) const EVIDENCE_FILE: &str = "evidence.bin";
pub(crate) const FINGERPRINT_FILE: &str = "fingerprint.bin";
/// Owned MPHF — used only at build time (construction + store).
pub(crate) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
/// Zero-copy MPHF for querying — ε-deserialized view into a memory-mapped file.
/// `MemCase` owns the mmap backing; `'static` is sound because MemCase pins the memory.
type MphfEps = PtrHash<u64, CubicEps, CachelineEfVec<&'static [CachelineEf]>, Xx64, &'static [u8]>;
/// The layer MPHF: `Xxh3Int` hasher, `CubicEps` buckets, multi-part (`SINGLE_PART = false`),
/// minimal (`REMAP = true`). Built owned (construction + store); queried through
/// `MemCase<Mphf>::uncase()`, the ε-copy view into the memory-mapped file.
pub(crate) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
// ── LayerEvidence ─────────────────────────────────────────────────────────────
@@ -158,7 +156,7 @@ pub struct MphfLayer {
/// not a bug).
unitigs: Arc<UnitigFileReader>,
mphf: MemCase<MphfEps>,
mphf: MemCase<Mphf>,
ev: OpenLayerEvidence,
n: usize,
}
@@ -171,7 +169,7 @@ impl MphfLayer {
/// not stored: `KmerLayer`, one level up, is what owns a layer's
/// directory as a persistent field, not this type.
pub fn open(dir: &Path) -> OKIResult<Self> {
let mphf: MemCase<MphfEps> = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty())
let mphf: MemCase<Mphf> = unsafe { Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) }
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
let unitigs = Arc::new(UnitigFileReader::open(&dir.join(UNITIGS_FILE))?);
let ev = LayerEvidence::at(dir)?.open()?;
@@ -191,7 +189,7 @@ impl MphfLayer {
/// - Hybrid: fingerprint check (fast path), zero false positives via `find_strict`.
#[inline]
pub fn find(&self, kmer: CanonicalKmer) -> Option<usize> {
let slot = self.mphf.index(&kmer.raw());
let slot = self.mphf.uncase().index(&kmer.raw());
if slot >= self.n {
return None;
}
@@ -221,7 +219,7 @@ impl MphfLayer {
/// - Approx: O(n) sequential scan of `unitigs.bin` to confirm the kmer
/// that owns the slot, then exact comparison.
pub fn find_strict(&self, kmer: CanonicalKmer) -> Option<usize> {
let slot = self.mphf.index(&kmer.raw());
let slot = self.mphf.uncase().index(&kmer.raw());
if slot >= self.n {
return None;
}
@@ -243,7 +241,7 @@ impl MphfLayer {
// the second file-open + mmap + kmer-count scan a fresh
// `open_sequential` call would repeat on every call.
for (stored, _, _) in self.unitigs.iter_indexed_canonical_kmers() {
if self.mphf.index(&stored.raw()) == slot {
if self.mphf.uncase().index(&stored.raw()) == slot {
return if stored == kmer { Some(slot) } else { None };
}
}
@@ -274,7 +272,7 @@ impl MphfLayer {
/// when the kmer is actually present in the layer; callers that need an
/// existence test should use [`find`](Self::find) instead.
pub fn hash(&self, kmer: CanonicalKmer) -> usize {
self.mphf.index(&kmer.raw())
self.mphf.uncase().index(&kmer.raw())
}
/// Batch raw MPHF lookup: kmers → slots.
@@ -283,7 +281,7 @@ impl MphfLayer {
/// input slice. Like [`index`](Self::index), no membership check is
/// performed.
pub fn hash_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
kmers.iter().map(|k| self.mphf.index(&k.raw())).collect()
kmers.iter().map(|k| self.mphf.uncase().index(&k.raw())).collect()
}
/// Iterate over all canonical kmers in the layer, in deterministic order.
@@ -426,11 +424,11 @@ impl Iterator for KmerBatchIter {
///
/// Use this when the caller guarantees that all queried kmers are in the MPHF
/// domain (e.g. when iterating the source's own unitigs during merge).
pub struct MphfOnly(MemCase<MphfEps>);
pub struct MphfOnly(MemCase<Mphf>);
impl MphfOnly {
pub fn open(dir: &Path) -> OKIResult<Self> {
let mphf: MemCase<MphfEps> = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty())
let mphf: MemCase<Mphf> = unsafe { Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) }
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
Ok(Self(mphf))
}
@@ -438,7 +436,7 @@ impl MphfOnly {
/// Return the slot for `kmer`. Only valid when `kmer` is in the MPHF domain.
#[inline]
pub fn index(&self, kmer: CanonicalKmer) -> usize {
self.0.index(&kmer.raw())
self.0.uncase().index(&kmer.raw())
}
/// Batch form of [`index`](Self::index) — one MPHF lookup pass over the
@@ -468,7 +466,7 @@ impl MphfLayer {
return Ok(0);
}
let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE))
let mphf: Mphf = unsafe { Mphf::load_full(&dir.join(MPHF_FILE)) }
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
let mut ev = EvidenceWriter::new(n);
@@ -512,7 +510,7 @@ impl MphfLayer {
return Ok(0);
}
let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE))
let mphf: Mphf = unsafe { Mphf::load_full(&dir.join(MPHF_FILE)) }
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
let mut fw = FingerprintVecWriter::new(n, b);
@@ -552,9 +550,9 @@ impl MphfLayer {
// ── Empty layer ───────────────────────────────────────────────────────
if n == 0 {
let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::<CubicEps>::default())
let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::<CubicEps>::default_balanced())
.ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?;
mphf.store(&dir.join(MPHF_FILE))
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
match mode {
IndexMode::Exact | IndexMode::Hybrid { .. } => {
@@ -576,9 +574,9 @@ impl MphfLayer {
let mphf: Mphf = Mphf::new_from_par_iter(
n,
keys.map(|k| k.raw()).par_bridge(),
PtrHashParams::<CubicEps>::default(),
PtrHashParams::<CubicEps>::default_balanced(),
);
mphf.store(&dir.join(MPHF_FILE))
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
// ── Pass 2: fill evidence files + callback ────────────────────────────
+3 -3
View File
@@ -14,9 +14,9 @@ obicompactvec = { path = "../obicompactvec" }
obipipeline = { path = "../obipipeline" }
obiread = { path = "../obiread" }
obiskbuilder = { path = "../obiskbuilder" }
cacheline-ef = "1.1"
epserde = "0.8"
ptr_hash = "1.1"
cacheline-ef = "2.0"
epserde = "0.13"
ptr_hash = { version = "2.1", features = ["epserde"] }
niffler = "3.0.0"
memmap2 = "0.9.10"
rayon = "1"
@@ -8,12 +8,12 @@ use epserde::ser::Serialize as EpSerialize;
use memmap2::Mmap;
use obicompactvec::PersistentCompactIntVecBuilder;
use obiskio::{SKFileReader, SKResult};
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64};
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int};
use tracing::debug;
use super::kmer_sort::sort_unique_kmers;
pub(super) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
pub(super) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
let file = fs::File::open(unique_path)?;
@@ -24,7 +24,7 @@ fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
// Sequential constructor: the outer par_iter over partitions already saturates
// the Rayon pool. new_from_par_iter would get no additional threads and adds
// coordination overhead. try_new accesses the same mmap'd pages at zero extra cost.
Mphf::try_new(kmers, PtrHashParams::<CubicEps>::default())
Mphf::try_new(kmers, PtrHashParams::<CubicEps>::default_balanced())
.ok_or_else(|| io::Error::other("ptr_hash construction failed"))
}
@@ -73,7 +73,7 @@ pub(super) fn count_partition(dir: &Path, dedup_path: &Path, chunk_kmers: usize)
)
.map_err(io::Error::other)?;
EpSerialize::store(&mphf, &dir.join("mphf1.bin"))
unsafe { EpSerialize::store(&mphf, &dir.join("mphf1.bin")) }
.map_err(|e| io::Error::other(e.to_string()))?;
Ok(())
@@ -33,9 +33,9 @@ use obikindex::layer::IndexMode;
use obikindex::layer::{KmerLayer, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xxh3Int};
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
pub(crate) trait PrivateBuilder {
/// Mark scatter as complete (`IndexState::Scattered`).
@@ -157,7 +157,7 @@ impl PrivateBuilder for KmerIndex {
let mphf1_opt: Option<Mphf> = if need_counts {
let p = layer0_dir.join("mphf1.bin");
p.exists().then(|| Mphf::load_full(&p).ok()).flatten()
p.exists().then(|| unsafe { Mphf::load_full(&p) }.ok()).flatten()
} else {
None
};