From d686d3950608e838ec2fefe84c35dfd7f51725f0 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Wed, 14 Oct 2026 18:55:44 +0200 Subject: [PATCH] Refactor indexing logic and update core dependencies This commit introduces significant updates to the indexing system, including the introduction of zero-copy types, modifications to the `Mphf` type definition, and new hybrid search strategies. Core algorithms are updated to handle new hashing types and improved error handling during pointer hash construction. Dependency versions are also updated across the board, including new security features and system-level crates. --- src/Cargo.lock | 320 +++++++++--------- src/obikindex/Cargo.toml | 6 +- src/obikindex/src/layer/mphf_layer.rs | 44 ++- src/obikindexer/Cargo.toml | 6 +- .../src/algorithms/counter/count.rs | 8 +- .../src/extensions/private_builder.rs | 6 +- 6 files changed, 185 insertions(+), 205 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index c2816d1a..545fb99c 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -30,6 +30,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "allocator-api2" version = "0.2.21" @@ -167,12 +173,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.11.1" @@ -191,6 +191,15 @@ dependencies = [ "wyz", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -268,11 +277,11 @@ dependencies = [ [[package]] name = "cacheline-ef" -version = "1.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af737c6c59cb018ecbe6472cbdf86d39c59d78252febfe311953a991b6e4ed85" +checksum = "6e701af8ea9f58c4ba18793a5d5dffd791cda99bf766976c59c583939d180fd5" dependencies = [ - "common_traits 0.11.4", + "common_traits", "epserde", "mem_dbg", ] @@ -312,6 +321,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chacha20" version = "0.10.1" @@ -319,7 +334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -433,17 +448,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "common_traits" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda9ae1f26adcae83adb2e92f69cf59421f2a277a942f49f8e59f2fcbd7cf062" -dependencies = [ - "anyhow", - "half", - "impl-tools 0.10.3", -] - [[package]] name = "common_traits" version = "0.13.0" @@ -452,7 +456,7 @@ checksum = "d8e0d7d0c5977b1e87f759d406740c38227125f374f54d81bc197e877f2c5f26" dependencies = [ "anyhow", "half", - "impl-tools 0.12.0", + "impl-tools", ] [[package]] @@ -482,6 +486,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -592,6 +605,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -645,15 +668,25 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "const-oid", - "crypto-common", + "crypto-common 0.2.2", ] [[package]] @@ -662,7 +695,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.11.1", + "bitflags", "objc2", ] @@ -698,27 +731,27 @@ dependencies = [ [[package]] name = "epserde" -version = "0.8.0" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c40d342ff20a2ce62d9a85ce406e672dfa137f902ac9670034533184f1533976" +checksum = "d6cd9f3212ce43e7b594a974c6e8cef4e11c6473d76b7b97cb01c239130789eb" dependencies = [ + "aliasable", "anyhow", - "bitflags 2.11.1", - "common_traits 0.11.4", + "bitflags", "epserde-derive", - "maligned", + "maybe-dangling", "mem_dbg", "mmap-rs", "sealed", + "sha2", "thiserror 2.0.18", - "xxhash-rust", ] [[package]] name = "epserde-derive" -version = "0.8.0" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac80cc78b69765703f48ad93f33b8919cf5d907cda7459ad6ba2919cbbe605dd" +checksum = "d73e3083270b2ff144c990e26580374fdfd512f61dd5a12da9b5bd749a7585e0" dependencies = [ "proc-macro2", "quote", @@ -806,6 +839,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -852,6 +895,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -899,7 +953,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c2e65a48d3b300843ac84a2fe8e166bb5a5b00f30054593bcee8157e4b465fd" dependencies = [ "arrayvec", - "bitflags 2.11.1", + "bitflags", "derive_more", "errno", "hwlocality-sys", @@ -935,18 +989,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "impl-tools" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ae95c9095c2f1126d7db785955c73cdc5fc33e7c3fa911bd4a42931672029a7" -dependencies = [ - "autocfg", - "impl-tools-lib 0.11.4", - "proc-macro-error2", - "syn 2.0.117", -] - [[package]] name = "impl-tools" version = "0.12.0" @@ -954,23 +996,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06c8567c75b167cdda5ecee024da9cc9581d41bd52460e2e0e768967972aeef1" dependencies = [ "autocfg", - "impl-tools-lib 0.12.0", + "impl-tools-lib", "proc-macro-error3", "syn 2.0.117", ] -[[package]] -name = "impl-tools-lib" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab699036df31c1f7d3561bfa6e9cb9bc3bb0fd2e2cd9bf121c31cb961d049ddf" -dependencies = [ - "proc-macro-error2", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "impl-tools-lib" version = "0.12.0" @@ -990,7 +1020,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -1043,9 +1073,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" dependencies = [ "either", ] @@ -1083,7 +1113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", ] [[package]] @@ -1157,7 +1187,7 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" dependencies = [ - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -1169,12 +1199,6 @@ dependencies = [ "libc", ] -[[package]] -name = "maligned" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e88c3cbe8288f77f293e48a28b3232e3defd203a6d839fa7f68ea4329e83464" - [[package]] name = "matchers" version = "0.2.0" @@ -1195,22 +1219,30 @@ dependencies = [ ] [[package]] -name = "mem_dbg" -version = "0.3.4" +name = "maybe-dangling" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728cc9dc97593cd22f7bc81fbef70a2d391d7a9a855e7d658b653318124a6cf0" +checksum = "59dbb09ed53f8e4f314e353dc6c1853ae5b4c480a668a422657804a544ea9f65" + +[[package]] +name = "mem_dbg" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b48a1086c746f4ee6ca5cb0acf856a14709bc4d2d20e03db150a12ddf2269e6d" dependencies = [ - "bitflags 2.11.1", - "maligned", + "aliasable", + "bitflags", + "hashbrown 0.16.1", + "maybe-dangling", "mem_dbg-derive", "mmap-rs", ] [[package]] name = "mem_dbg-derive" -version = "0.2.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d84f40c93b0508d5565db79a814d02d5b2545967205ce44be211592aafa34d6c" +checksum = "eb910efe8da52f13da727170e352e50a1764579a6fb1065d00d9556da19c79ac" dependencies = [ "proc-macro2", "quote", @@ -1232,15 +1264,6 @@ dependencies = [ "libc", ] -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1253,17 +1276,17 @@ dependencies = [ [[package]] name = "mmap-rs" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86968d85441db75203c34deefd0c88032f275aaa85cee19a1dcfff6ae9df56da" +checksum = "4ecce9d566cb9234ae3db9e249c8b55665feaaf32b0859ff1e27e310d2beb3d8" dependencies = [ - "bitflags 1.3.2", + "bitflags", "combine", "libc", "mach2", "nix", "sysctl", - "thiserror 1.0.69", + "thiserror 2.0.18", "widestring", "windows 0.48.0", ] @@ -1326,15 +1349,14 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.4" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 1.3.2", + "bitflags", "cfg-if", + "cfg_aliases", "libc", - "memoffset", - "pin-utils", ] [[package]] @@ -1386,7 +1408,7 @@ dependencies = [ name = "obicompactvec" version = "0.1.0" dependencies = [ - "common_traits 0.13.0", + "common_traits", "memmap2", "ndarray", "rayon", @@ -1399,7 +1421,7 @@ version = "0.1.0" dependencies = [ "ahash", "crossbeam-channel", - "hashbrown", + "hashbrown 0.17.1", "obifastwrite", "obikseq", "rayon", @@ -1773,7 +1795,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.11.1", + "bitflags", "dispatch2", "objc2", ] @@ -1790,7 +1812,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags 2.11.1", + "bitflags", "objc2", ] @@ -1884,12 +1906,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkg-config" version = "0.3.33" @@ -1949,14 +1965,10 @@ dependencies = [ ] [[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" +name = "prefetch-index" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" -dependencies = [ - "proc-macro2", - "quote", -] +checksum = "9057806a8d77d67bccdc0f542db43737a6f19ada3efab2adc63277feea27310f" [[package]] name = "proc-macro-error-attr3" @@ -1968,17 +1980,6 @@ dependencies = [ "quote", ] -[[package]] -name = "proc-macro-error2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" -dependencies = [ - "proc-macro-error-attr2", - "proc-macro2", - "quote", -] - [[package]] name = "proc-macro-error3" version = "3.1.0" @@ -2001,30 +2002,25 @@ dependencies = [ [[package]] name = "ptr_hash" -version = "1.1.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b4e4fb9c4c2ba3e5b060f53ef46afd3de37345b08e3ec0f2c65e0ca1d57ccbd" +checksum = "b22cf1be774892680e056d3387db0dcaa4fd46964b270671ef661e127186b969" dependencies = [ - "anyhow", "bitvec", "cacheline-ef", - "clap", "colored", - "common_traits 0.11.4", "epserde", "epserde-derive", "fastrand", "fxhash", - "itertools 0.14.0", - "lazy_static", + "itertools 0.15.0", "log", "mem_dbg", - "rand 0.9.4", - "rand_chacha 0.9.0", + "prefetch-index", + "rand 0.10.2", + "rand_chacha 0.10.0", "rayon", "rdst", - "rustc-hash", - "serde", "sucds", "tempfile", "xxhash-rust", @@ -2097,16 +2093,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rand" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", -] - [[package]] name = "rand" version = "0.10.2" @@ -2130,12 +2116,12 @@ dependencies = [ [[package]] name = "rand_chacha" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" dependencies = [ "ppv-lite86", - "rand_core 0.9.5", + "rand_core 0.10.1", ] [[package]] @@ -2147,15 +2133,6 @@ dependencies = [ "getrandom 0.2.17", ] -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - [[package]] name = "rand_core" version = "0.10.1" @@ -2215,7 +2192,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.1", + "bitflags", ] [[package]] @@ -2278,7 +2255,7 @@ checksum = "d9776093b7ca170454ab1406954f7b7d97a57c51dc6c0642957fb2ef25c2d399" dependencies = [ "bytecheck", "bytes", - "hashbrown", + "hashbrown 0.17.1", "indexmap", "munge", "ptr_meta", @@ -2300,12 +2277,6 @@ dependencies = [ "syn 3.0.3", ] -[[package]] -name = "rustc-hash" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" - [[package]] name = "rustc_version" version = "0.4.1" @@ -2321,7 +2292,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -2463,13 +2434,24 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha3" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" dependencies = [ - "digest", + "digest 0.11.3", "keccak", ] @@ -2592,11 +2574,11 @@ dependencies = [ [[package]] name = "sysctl" -version = "0.5.5" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" +checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.11.1", + "bitflags", "byteorder", "enum-as-inner", "libc", diff --git a/src/obikindex/Cargo.toml b/src/obikindex/Cargo.toml index 23e8ac61..5153655b 100644 --- a/src/obikindex/Cargo.toml +++ b/src/obikindex/Cargo.toml @@ -12,9 +12,9 @@ obicompactvec = { path = "../obicompactvec" } obidebruinj = { path = "../obidebruinj" } obipipeline = { path = "../obipipeline" } obikentropy = { path = "../obikentropy" } -cacheline-ef = "1.1" -epserde = "0.8" -ptr_hash = "1.1" +cacheline-ef = "2.0" +epserde = "0.13" +ptr_hash = { version = "2.1", features = ["epserde"] } niffler = "3.0.0" memmap2 = "0.9.10" ndarray = "0.17" diff --git a/src/obikindex/src/layer/mphf_layer.rs b/src/obikindex/src/layer/mphf_layer.rs index 0ce61819..5932503c 100644 --- a/src/obikindex/src/layer/mphf_layer.rs +++ b/src/obikindex/src/layer/mphf_layer.rs @@ -8,7 +8,7 @@ use epserde::prelude::*; use obicompactvec::LayerMeta; use obikseq::CanonicalKmer; use obiskio::{CanonicalKmerIter, UnitigFileReader, UnitigFileWriter, build_unitig_idx}; -use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64}; +use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int}; use crate::index::error::{OKIError, OKIResult}; use crate::layer::evidence::{Evidence, EvidenceWriter}; @@ -20,12 +20,10 @@ pub(crate) const UNITIGS_FILE: &str = "unitigs.bin"; pub(crate) const EVIDENCE_FILE: &str = "evidence.bin"; pub(crate) const FINGERPRINT_FILE: &str = "fingerprint.bin"; -/// Owned MPHF — used only at build time (construction + store). -pub(crate) type Mphf = PtrHash>, Xx64, Vec>; - -/// Zero-copy MPHF for querying — ε-deserialized view into a memory-mapped file. -/// `MemCase` owns the mmap backing; `'static` is sound because MemCase pins the memory. -type MphfEps = PtrHash, Xx64, &'static [u8]>; +/// The layer MPHF: `Xxh3Int` hasher, `CubicEps` buckets, multi-part (`SINGLE_PART = false`), +/// minimal (`REMAP = true`). Built owned (construction + store); queried through +/// `MemCase::uncase()`, the ε-copy view into the memory-mapped file. +pub(crate) type Mphf = PtrHash>, Xxh3Int, Vec, false, true>; // ── LayerEvidence ───────────────────────────────────────────────────────────── @@ -158,7 +156,7 @@ pub struct MphfLayer { /// not a bug). unitigs: Arc, - mphf: MemCase, + mphf: MemCase, ev: OpenLayerEvidence, n: usize, } @@ -171,7 +169,7 @@ impl MphfLayer { /// not stored: `KmerLayer`, one level up, is what owns a layer's /// directory as a persistent field, not this type. pub fn open(dir: &Path) -> OKIResult { - let mphf: MemCase = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) + let mphf: MemCase = unsafe { Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; let unitigs = Arc::new(UnitigFileReader::open(&dir.join(UNITIGS_FILE))?); let ev = LayerEvidence::at(dir)?.open()?; @@ -191,7 +189,7 @@ impl MphfLayer { /// - Hybrid: fingerprint check (fast path), zero false positives via `find_strict`. #[inline] pub fn find(&self, kmer: CanonicalKmer) -> Option { - let slot = self.mphf.index(&kmer.raw()); + let slot = self.mphf.uncase().index(&kmer.raw()); if slot >= self.n { return None; } @@ -221,7 +219,7 @@ impl MphfLayer { /// - Approx: O(n) sequential scan of `unitigs.bin` to confirm the kmer /// that owns the slot, then exact comparison. pub fn find_strict(&self, kmer: CanonicalKmer) -> Option { - let slot = self.mphf.index(&kmer.raw()); + let slot = self.mphf.uncase().index(&kmer.raw()); if slot >= self.n { return None; } @@ -243,7 +241,7 @@ impl MphfLayer { // the second file-open + mmap + kmer-count scan a fresh // `open_sequential` call would repeat on every call. for (stored, _, _) in self.unitigs.iter_indexed_canonical_kmers() { - if self.mphf.index(&stored.raw()) == slot { + if self.mphf.uncase().index(&stored.raw()) == slot { return if stored == kmer { Some(slot) } else { None }; } } @@ -274,7 +272,7 @@ impl MphfLayer { /// when the kmer is actually present in the layer; callers that need an /// existence test should use [`find`](Self::find) instead. pub fn hash(&self, kmer: CanonicalKmer) -> usize { - self.mphf.index(&kmer.raw()) + self.mphf.uncase().index(&kmer.raw()) } /// Batch raw MPHF lookup: kmers → slots. @@ -283,7 +281,7 @@ impl MphfLayer { /// input slice. Like [`index`](Self::index), no membership check is /// performed. pub fn hash_batch(&self, kmers: &[CanonicalKmer]) -> Vec { - kmers.iter().map(|k| self.mphf.index(&k.raw())).collect() + kmers.iter().map(|k| self.mphf.uncase().index(&k.raw())).collect() } /// Iterate over all canonical kmers in the layer, in deterministic order. @@ -426,11 +424,11 @@ impl Iterator for KmerBatchIter { /// /// Use this when the caller guarantees that all queried kmers are in the MPHF /// domain (e.g. when iterating the source's own unitigs during merge). -pub struct MphfOnly(MemCase); +pub struct MphfOnly(MemCase); impl MphfOnly { pub fn open(dir: &Path) -> OKIResult { - let mphf: MemCase = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) + let mphf: MemCase = unsafe { Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; Ok(Self(mphf)) } @@ -438,7 +436,7 @@ impl MphfOnly { /// Return the slot for `kmer`. Only valid when `kmer` is in the MPHF domain. #[inline] pub fn index(&self, kmer: CanonicalKmer) -> usize { - self.0.index(&kmer.raw()) + self.0.uncase().index(&kmer.raw()) } /// Batch form of [`index`](Self::index) — one MPHF lookup pass over the @@ -468,7 +466,7 @@ impl MphfLayer { return Ok(0); } - let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE)) + let mphf: Mphf = unsafe { Mphf::load_full(&dir.join(MPHF_FILE)) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; let mut ev = EvidenceWriter::new(n); @@ -512,7 +510,7 @@ impl MphfLayer { return Ok(0); } - let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE)) + let mphf: Mphf = unsafe { Mphf::load_full(&dir.join(MPHF_FILE)) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; let mut fw = FingerprintVecWriter::new(n, b); @@ -552,9 +550,9 @@ impl MphfLayer { // ── Empty layer ─────────────────────────────────────────────────────── if n == 0 { - let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::::default()) + let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::::default_balanced()) .ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?; - mphf.store(&dir.join(MPHF_FILE)) + unsafe { mphf.store(&dir.join(MPHF_FILE)) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; match mode { IndexMode::Exact | IndexMode::Hybrid { .. } => { @@ -576,9 +574,9 @@ impl MphfLayer { let mphf: Mphf = Mphf::new_from_par_iter( n, keys.map(|k| k.raw()).par_bridge(), - PtrHashParams::::default(), + PtrHashParams::::default_balanced(), ); - mphf.store(&dir.join(MPHF_FILE)) + unsafe { mphf.store(&dir.join(MPHF_FILE)) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; // ── Pass 2: fill evidence files + callback ──────────────────────────── diff --git a/src/obikindexer/Cargo.toml b/src/obikindexer/Cargo.toml index 935268b3..af4b88a7 100644 --- a/src/obikindexer/Cargo.toml +++ b/src/obikindexer/Cargo.toml @@ -14,9 +14,9 @@ obicompactvec = { path = "../obicompactvec" } obipipeline = { path = "../obipipeline" } obiread = { path = "../obiread" } obiskbuilder = { path = "../obiskbuilder" } -cacheline-ef = "1.1" -epserde = "0.8" -ptr_hash = "1.1" +cacheline-ef = "2.0" +epserde = "0.13" +ptr_hash = { version = "2.1", features = ["epserde"] } niffler = "3.0.0" memmap2 = "0.9.10" rayon = "1" diff --git a/src/obikindexer/src/algorithms/counter/count.rs b/src/obikindexer/src/algorithms/counter/count.rs index 6d5a9990..a9332403 100644 --- a/src/obikindexer/src/algorithms/counter/count.rs +++ b/src/obikindexer/src/algorithms/counter/count.rs @@ -8,12 +8,12 @@ use epserde::ser::Serialize as EpSerialize; use memmap2::Mmap; use obicompactvec::PersistentCompactIntVecBuilder; use obiskio::{SKFileReader, SKResult}; -use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64}; +use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int}; use tracing::debug; use super::kmer_sort::sort_unique_kmers; -pub(super) type Mphf = PtrHash>, Xx64, Vec>; +pub(super) type Mphf = PtrHash>, Xxh3Int, Vec, false, true>; fn build_mphf(unique_path: &Path, f0: usize) -> io::Result { let file = fs::File::open(unique_path)?; @@ -24,7 +24,7 @@ fn build_mphf(unique_path: &Path, f0: usize) -> io::Result { // Sequential constructor: the outer par_iter over partitions already saturates // the Rayon pool. new_from_par_iter would get no additional threads and adds // coordination overhead. try_new accesses the same mmap'd pages at zero extra cost. - Mphf::try_new(kmers, PtrHashParams::::default()) + Mphf::try_new(kmers, PtrHashParams::::default_balanced()) .ok_or_else(|| io::Error::other("ptr_hash construction failed")) } @@ -73,7 +73,7 @@ pub(super) fn count_partition(dir: &Path, dedup_path: &Path, chunk_kmers: usize) ) .map_err(io::Error::other)?; - EpSerialize::store(&mphf, &dir.join("mphf1.bin")) + unsafe { EpSerialize::store(&mphf, &dir.join("mphf1.bin")) } .map_err(|e| io::Error::other(e.to_string()))?; Ok(()) diff --git a/src/obikindexer/src/extensions/private_builder.rs b/src/obikindexer/src/extensions/private_builder.rs index 661d9194..c4c224a0 100644 --- a/src/obikindexer/src/extensions/private_builder.rs +++ b/src/obikindexer/src/extensions/private_builder.rs @@ -33,9 +33,9 @@ use obikindex::layer::IndexMode; use obikindex::layer::{KmerLayer, TypedLayer}; use obikindex::{KmerIndex, OKIError, OKIResult}; use obiskio::{SKError, SKFileMeta, SKFileReader}; -use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64}; +use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xxh3Int}; -type Mphf = PtrHash>, Xx64, Vec>; +type Mphf = PtrHash>, Xxh3Int, Vec, false, true>; pub(crate) trait PrivateBuilder { /// Mark scatter as complete (`IndexState::Scattered`). @@ -157,7 +157,7 @@ impl PrivateBuilder for KmerIndex { let mphf1_opt: Option = if need_counts { let p = layer0_dir.join("mphf1.bin"); - p.exists().then(|| Mphf::load_full(&p).ok()).flatten() + p.exists().then(|| unsafe { Mphf::load_full(&p) }.ok()).flatten() } else { None };