diff --git a/src/Cargo.lock b/src/Cargo.lock index c2816d1a..545fb99c 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -30,6 +30,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "allocator-api2" version = "0.2.21" @@ -167,12 +173,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.11.1" @@ -191,6 +191,15 @@ dependencies = [ "wyz", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -268,11 +277,11 @@ dependencies = [ [[package]] name = "cacheline-ef" -version = "1.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af737c6c59cb018ecbe6472cbdf86d39c59d78252febfe311953a991b6e4ed85" +checksum = "6e701af8ea9f58c4ba18793a5d5dffd791cda99bf766976c59c583939d180fd5" dependencies = [ - "common_traits 0.11.4", + "common_traits", "epserde", "mem_dbg", ] @@ -312,6 +321,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chacha20" version = "0.10.1" @@ -319,7 +334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -433,17 +448,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "common_traits" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda9ae1f26adcae83adb2e92f69cf59421f2a277a942f49f8e59f2fcbd7cf062" -dependencies = [ - "anyhow", - "half", - "impl-tools 0.10.3", -] - [[package]] name = "common_traits" version = "0.13.0" @@ -452,7 +456,7 @@ checksum = "d8e0d7d0c5977b1e87f759d406740c38227125f374f54d81bc197e877f2c5f26" dependencies = [ "anyhow", "half", - "impl-tools 0.12.0", + "impl-tools", ] [[package]] @@ -482,6 +486,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -592,6 +605,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -645,15 +668,25 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "const-oid", - "crypto-common", + "crypto-common 0.2.2", ] [[package]] @@ -662,7 +695,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.11.1", + "bitflags", "objc2", ] @@ -698,27 +731,27 @@ dependencies = [ [[package]] name = "epserde" -version = "0.8.0" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c40d342ff20a2ce62d9a85ce406e672dfa137f902ac9670034533184f1533976" +checksum = "d6cd9f3212ce43e7b594a974c6e8cef4e11c6473d76b7b97cb01c239130789eb" dependencies = [ + "aliasable", "anyhow", - "bitflags 2.11.1", - "common_traits 0.11.4", + "bitflags", "epserde-derive", - "maligned", + "maybe-dangling", "mem_dbg", "mmap-rs", "sealed", + "sha2", "thiserror 2.0.18", - "xxhash-rust", ] [[package]] name = "epserde-derive" -version = "0.8.0" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac80cc78b69765703f48ad93f33b8919cf5d907cda7459ad6ba2919cbbe605dd" +checksum = "d73e3083270b2ff144c990e26580374fdfd512f61dd5a12da9b5bd749a7585e0" dependencies = [ "proc-macro2", "quote", @@ -806,6 +839,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -852,6 +895,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -899,7 +953,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c2e65a48d3b300843ac84a2fe8e166bb5a5b00f30054593bcee8157e4b465fd" dependencies = [ "arrayvec", - "bitflags 2.11.1", + "bitflags", "derive_more", "errno", "hwlocality-sys", @@ -935,18 +989,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "impl-tools" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ae95c9095c2f1126d7db785955c73cdc5fc33e7c3fa911bd4a42931672029a7" -dependencies = [ - "autocfg", - "impl-tools-lib 0.11.4", - "proc-macro-error2", - "syn 2.0.117", -] - [[package]] name = "impl-tools" version = "0.12.0" @@ -954,23 +996,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06c8567c75b167cdda5ecee024da9cc9581d41bd52460e2e0e768967972aeef1" dependencies = [ "autocfg", - "impl-tools-lib 0.12.0", + "impl-tools-lib", "proc-macro-error3", "syn 2.0.117", ] -[[package]] -name = "impl-tools-lib" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab699036df31c1f7d3561bfa6e9cb9bc3bb0fd2e2cd9bf121c31cb961d049ddf" -dependencies = [ - "proc-macro-error2", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "impl-tools-lib" version = "0.12.0" @@ -990,7 +1020,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -1043,9 +1073,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" dependencies = [ "either", ] @@ -1083,7 +1113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", ] [[package]] @@ -1157,7 +1187,7 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" dependencies = [ - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -1169,12 +1199,6 @@ dependencies = [ "libc", ] -[[package]] -name = "maligned" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e88c3cbe8288f77f293e48a28b3232e3defd203a6d839fa7f68ea4329e83464" - [[package]] name = "matchers" version = "0.2.0" @@ -1195,22 +1219,30 @@ dependencies = [ ] [[package]] -name = "mem_dbg" -version = "0.3.4" +name = "maybe-dangling" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728cc9dc97593cd22f7bc81fbef70a2d391d7a9a855e7d658b653318124a6cf0" +checksum = "59dbb09ed53f8e4f314e353dc6c1853ae5b4c480a668a422657804a544ea9f65" + +[[package]] +name = "mem_dbg" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b48a1086c746f4ee6ca5cb0acf856a14709bc4d2d20e03db150a12ddf2269e6d" dependencies = [ - "bitflags 2.11.1", - "maligned", + "aliasable", + "bitflags", + "hashbrown 0.16.1", + "maybe-dangling", "mem_dbg-derive", "mmap-rs", ] [[package]] name = "mem_dbg-derive" -version = "0.2.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d84f40c93b0508d5565db79a814d02d5b2545967205ce44be211592aafa34d6c" +checksum = "eb910efe8da52f13da727170e352e50a1764579a6fb1065d00d9556da19c79ac" dependencies = [ "proc-macro2", "quote", @@ -1232,15 +1264,6 @@ dependencies = [ "libc", ] -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1253,17 +1276,17 @@ dependencies = [ [[package]] name = "mmap-rs" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86968d85441db75203c34deefd0c88032f275aaa85cee19a1dcfff6ae9df56da" +checksum = "4ecce9d566cb9234ae3db9e249c8b55665feaaf32b0859ff1e27e310d2beb3d8" dependencies = [ - "bitflags 1.3.2", + "bitflags", "combine", "libc", "mach2", "nix", "sysctl", - "thiserror 1.0.69", + "thiserror 2.0.18", "widestring", "windows 0.48.0", ] @@ -1326,15 +1349,14 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.4" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 1.3.2", + "bitflags", "cfg-if", + "cfg_aliases", "libc", - "memoffset", - "pin-utils", ] [[package]] @@ -1386,7 +1408,7 @@ dependencies = [ name = "obicompactvec" version = "0.1.0" dependencies = [ - "common_traits 0.13.0", + "common_traits", "memmap2", "ndarray", "rayon", @@ -1399,7 +1421,7 @@ version = "0.1.0" dependencies = [ "ahash", "crossbeam-channel", - "hashbrown", + "hashbrown 0.17.1", "obifastwrite", "obikseq", "rayon", @@ -1773,7 +1795,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.11.1", + "bitflags", "dispatch2", "objc2", ] @@ -1790,7 +1812,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags 2.11.1", + "bitflags", "objc2", ] @@ -1884,12 +1906,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkg-config" version = "0.3.33" @@ -1949,14 +1965,10 @@ dependencies = [ ] [[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" +name = "prefetch-index" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" -dependencies = [ - "proc-macro2", - "quote", -] +checksum = "9057806a8d77d67bccdc0f542db43737a6f19ada3efab2adc63277feea27310f" [[package]] name = "proc-macro-error-attr3" @@ -1968,17 +1980,6 @@ dependencies = [ "quote", ] -[[package]] -name = "proc-macro-error2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" -dependencies = [ - "proc-macro-error-attr2", - "proc-macro2", - "quote", -] - [[package]] name = "proc-macro-error3" version = "3.1.0" @@ -2001,30 +2002,25 @@ dependencies = [ [[package]] name = "ptr_hash" -version = "1.1.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b4e4fb9c4c2ba3e5b060f53ef46afd3de37345b08e3ec0f2c65e0ca1d57ccbd" +checksum = "b22cf1be774892680e056d3387db0dcaa4fd46964b270671ef661e127186b969" dependencies = [ - "anyhow", "bitvec", "cacheline-ef", - "clap", "colored", - "common_traits 0.11.4", "epserde", "epserde-derive", "fastrand", "fxhash", - "itertools 0.14.0", - "lazy_static", + "itertools 0.15.0", "log", "mem_dbg", - "rand 0.9.4", - "rand_chacha 0.9.0", + "prefetch-index", + "rand 0.10.2", + "rand_chacha 0.10.0", "rayon", "rdst", - "rustc-hash", - "serde", "sucds", "tempfile", "xxhash-rust", @@ -2097,16 +2093,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rand" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", -] - [[package]] name = "rand" version = "0.10.2" @@ -2130,12 +2116,12 @@ dependencies = [ [[package]] name = "rand_chacha" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" dependencies = [ "ppv-lite86", - "rand_core 0.9.5", + "rand_core 0.10.1", ] [[package]] @@ -2147,15 +2133,6 @@ dependencies = [ "getrandom 0.2.17", ] -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - [[package]] name = "rand_core" version = "0.10.1" @@ -2215,7 +2192,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.1", + "bitflags", ] [[package]] @@ -2278,7 +2255,7 @@ checksum = "d9776093b7ca170454ab1406954f7b7d97a57c51dc6c0642957fb2ef25c2d399" dependencies = [ "bytecheck", "bytes", - "hashbrown", + "hashbrown 0.17.1", "indexmap", "munge", "ptr_meta", @@ -2300,12 +2277,6 @@ dependencies = [ "syn 3.0.3", ] -[[package]] -name = "rustc-hash" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" - [[package]] name = "rustc_version" version = "0.4.1" @@ -2321,7 +2292,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -2463,13 +2434,24 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha3" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" dependencies = [ - "digest", + "digest 0.11.3", "keccak", ] @@ -2592,11 +2574,11 @@ dependencies = [ [[package]] name = "sysctl" -version = "0.5.5" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" +checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.11.1", + "bitflags", "byteorder", "enum-as-inner", "libc", diff --git a/src/obikindex/Cargo.toml b/src/obikindex/Cargo.toml index 23e8ac61..5153655b 100644 --- a/src/obikindex/Cargo.toml +++ b/src/obikindex/Cargo.toml @@ -12,9 +12,9 @@ obicompactvec = { path = "../obicompactvec" } obidebruinj = { path = "../obidebruinj" } obipipeline = { path = "../obipipeline" } obikentropy = { path = "../obikentropy" } -cacheline-ef = "1.1" -epserde = "0.8" -ptr_hash = "1.1" +cacheline-ef = "2.0" +epserde = "0.13" +ptr_hash = { version = "2.1", features = ["epserde"] } niffler = "3.0.0" memmap2 = "0.9.10" ndarray = "0.17" diff --git a/src/obikindex/src/layer/mphf_layer.rs b/src/obikindex/src/layer/mphf_layer.rs index 0ce61819..5932503c 100644 --- a/src/obikindex/src/layer/mphf_layer.rs +++ b/src/obikindex/src/layer/mphf_layer.rs @@ -8,7 +8,7 @@ use epserde::prelude::*; use obicompactvec::LayerMeta; use obikseq::CanonicalKmer; use obiskio::{CanonicalKmerIter, UnitigFileReader, UnitigFileWriter, build_unitig_idx}; -use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64}; +use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int}; use crate::index::error::{OKIError, OKIResult}; use crate::layer::evidence::{Evidence, EvidenceWriter}; @@ -20,12 +20,10 @@ pub(crate) const UNITIGS_FILE: &str = "unitigs.bin"; pub(crate) const EVIDENCE_FILE: &str = "evidence.bin"; pub(crate) const FINGERPRINT_FILE: &str = "fingerprint.bin"; -/// Owned MPHF — used only at build time (construction + store). -pub(crate) type Mphf = PtrHash>, Xx64, Vec>; - -/// Zero-copy MPHF for querying — ε-deserialized view into a memory-mapped file. -/// `MemCase` owns the mmap backing; `'static` is sound because MemCase pins the memory. -type MphfEps = PtrHash, Xx64, &'static [u8]>; +/// The layer MPHF: `Xxh3Int` hasher, `CubicEps` buckets, multi-part (`SINGLE_PART = false`), +/// minimal (`REMAP = true`). Built owned (construction + store); queried through +/// `MemCase::uncase()`, the ε-copy view into the memory-mapped file. +pub(crate) type Mphf = PtrHash>, Xxh3Int, Vec, false, true>; // ── LayerEvidence ───────────────────────────────────────────────────────────── @@ -158,7 +156,7 @@ pub struct MphfLayer { /// not a bug). unitigs: Arc, - mphf: MemCase, + mphf: MemCase, ev: OpenLayerEvidence, n: usize, } @@ -171,7 +169,7 @@ impl MphfLayer { /// not stored: `KmerLayer`, one level up, is what owns a layer's /// directory as a persistent field, not this type. pub fn open(dir: &Path) -> OKIResult { - let mphf: MemCase = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) + let mphf: MemCase = unsafe { Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; let unitigs = Arc::new(UnitigFileReader::open(&dir.join(UNITIGS_FILE))?); let ev = LayerEvidence::at(dir)?.open()?; @@ -191,7 +189,7 @@ impl MphfLayer { /// - Hybrid: fingerprint check (fast path), zero false positives via `find_strict`. #[inline] pub fn find(&self, kmer: CanonicalKmer) -> Option { - let slot = self.mphf.index(&kmer.raw()); + let slot = self.mphf.uncase().index(&kmer.raw()); if slot >= self.n { return None; } @@ -221,7 +219,7 @@ impl MphfLayer { /// - Approx: O(n) sequential scan of `unitigs.bin` to confirm the kmer /// that owns the slot, then exact comparison. pub fn find_strict(&self, kmer: CanonicalKmer) -> Option { - let slot = self.mphf.index(&kmer.raw()); + let slot = self.mphf.uncase().index(&kmer.raw()); if slot >= self.n { return None; } @@ -243,7 +241,7 @@ impl MphfLayer { // the second file-open + mmap + kmer-count scan a fresh // `open_sequential` call would repeat on every call. for (stored, _, _) in self.unitigs.iter_indexed_canonical_kmers() { - if self.mphf.index(&stored.raw()) == slot { + if self.mphf.uncase().index(&stored.raw()) == slot { return if stored == kmer { Some(slot) } else { None }; } } @@ -274,7 +272,7 @@ impl MphfLayer { /// when the kmer is actually present in the layer; callers that need an /// existence test should use [`find`](Self::find) instead. pub fn hash(&self, kmer: CanonicalKmer) -> usize { - self.mphf.index(&kmer.raw()) + self.mphf.uncase().index(&kmer.raw()) } /// Batch raw MPHF lookup: kmers → slots. @@ -283,7 +281,7 @@ impl MphfLayer { /// input slice. Like [`index`](Self::index), no membership check is /// performed. pub fn hash_batch(&self, kmers: &[CanonicalKmer]) -> Vec { - kmers.iter().map(|k| self.mphf.index(&k.raw())).collect() + kmers.iter().map(|k| self.mphf.uncase().index(&k.raw())).collect() } /// Iterate over all canonical kmers in the layer, in deterministic order. @@ -426,11 +424,11 @@ impl Iterator for KmerBatchIter { /// /// Use this when the caller guarantees that all queried kmers are in the MPHF /// domain (e.g. when iterating the source's own unitigs during merge). -pub struct MphfOnly(MemCase); +pub struct MphfOnly(MemCase); impl MphfOnly { pub fn open(dir: &Path) -> OKIResult { - let mphf: MemCase = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) + let mphf: MemCase = unsafe { Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; Ok(Self(mphf)) } @@ -438,7 +436,7 @@ impl MphfOnly { /// Return the slot for `kmer`. Only valid when `kmer` is in the MPHF domain. #[inline] pub fn index(&self, kmer: CanonicalKmer) -> usize { - self.0.index(&kmer.raw()) + self.0.uncase().index(&kmer.raw()) } /// Batch form of [`index`](Self::index) — one MPHF lookup pass over the @@ -468,7 +466,7 @@ impl MphfLayer { return Ok(0); } - let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE)) + let mphf: Mphf = unsafe { Mphf::load_full(&dir.join(MPHF_FILE)) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; let mut ev = EvidenceWriter::new(n); @@ -512,7 +510,7 @@ impl MphfLayer { return Ok(0); } - let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE)) + let mphf: Mphf = unsafe { Mphf::load_full(&dir.join(MPHF_FILE)) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; let mut fw = FingerprintVecWriter::new(n, b); @@ -552,9 +550,9 @@ impl MphfLayer { // ── Empty layer ─────────────────────────────────────────────────────── if n == 0 { - let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::::default()) + let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::::default_balanced()) .ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?; - mphf.store(&dir.join(MPHF_FILE)) + unsafe { mphf.store(&dir.join(MPHF_FILE)) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; match mode { IndexMode::Exact | IndexMode::Hybrid { .. } => { @@ -576,9 +574,9 @@ impl MphfLayer { let mphf: Mphf = Mphf::new_from_par_iter( n, keys.map(|k| k.raw()).par_bridge(), - PtrHashParams::::default(), + PtrHashParams::::default_balanced(), ); - mphf.store(&dir.join(MPHF_FILE)) + unsafe { mphf.store(&dir.join(MPHF_FILE)) } .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; // ── Pass 2: fill evidence files + callback ──────────────────────────── diff --git a/src/obikindexer/Cargo.toml b/src/obikindexer/Cargo.toml index 935268b3..af4b88a7 100644 --- a/src/obikindexer/Cargo.toml +++ b/src/obikindexer/Cargo.toml @@ -14,9 +14,9 @@ obicompactvec = { path = "../obicompactvec" } obipipeline = { path = "../obipipeline" } obiread = { path = "../obiread" } obiskbuilder = { path = "../obiskbuilder" } -cacheline-ef = "1.1" -epserde = "0.8" -ptr_hash = "1.1" +cacheline-ef = "2.0" +epserde = "0.13" +ptr_hash = { version = "2.1", features = ["epserde"] } niffler = "3.0.0" memmap2 = "0.9.10" rayon = "1" diff --git a/src/obikindexer/src/algorithms/counter/count.rs b/src/obikindexer/src/algorithms/counter/count.rs index 6d5a9990..a9332403 100644 --- a/src/obikindexer/src/algorithms/counter/count.rs +++ b/src/obikindexer/src/algorithms/counter/count.rs @@ -8,12 +8,12 @@ use epserde::ser::Serialize as EpSerialize; use memmap2::Mmap; use obicompactvec::PersistentCompactIntVecBuilder; use obiskio::{SKFileReader, SKResult}; -use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64}; +use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int}; use tracing::debug; use super::kmer_sort::sort_unique_kmers; -pub(super) type Mphf = PtrHash>, Xx64, Vec>; +pub(super) type Mphf = PtrHash>, Xxh3Int, Vec, false, true>; fn build_mphf(unique_path: &Path, f0: usize) -> io::Result { let file = fs::File::open(unique_path)?; @@ -24,7 +24,7 @@ fn build_mphf(unique_path: &Path, f0: usize) -> io::Result { // Sequential constructor: the outer par_iter over partitions already saturates // the Rayon pool. new_from_par_iter would get no additional threads and adds // coordination overhead. try_new accesses the same mmap'd pages at zero extra cost. - Mphf::try_new(kmers, PtrHashParams::::default()) + Mphf::try_new(kmers, PtrHashParams::::default_balanced()) .ok_or_else(|| io::Error::other("ptr_hash construction failed")) } @@ -73,7 +73,7 @@ pub(super) fn count_partition(dir: &Path, dedup_path: &Path, chunk_kmers: usize) ) .map_err(io::Error::other)?; - EpSerialize::store(&mphf, &dir.join("mphf1.bin")) + unsafe { EpSerialize::store(&mphf, &dir.join("mphf1.bin")) } .map_err(|e| io::Error::other(e.to_string()))?; Ok(()) diff --git a/src/obikindexer/src/extensions/private_builder.rs b/src/obikindexer/src/extensions/private_builder.rs index 661d9194..c4c224a0 100644 --- a/src/obikindexer/src/extensions/private_builder.rs +++ b/src/obikindexer/src/extensions/private_builder.rs @@ -33,9 +33,9 @@ use obikindex::layer::IndexMode; use obikindex::layer::{KmerLayer, TypedLayer}; use obikindex::{KmerIndex, OKIError, OKIResult}; use obiskio::{SKError, SKFileMeta, SKFileReader}; -use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64}; +use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xxh3Int}; -type Mphf = PtrHash>, Xx64, Vec>; +type Mphf = PtrHash>, Xxh3Int, Vec, false, true>; pub(crate) trait PrivateBuilder { /// Mark scatter as complete (`IndexState::Scattered`). @@ -157,7 +157,7 @@ impl PrivateBuilder for KmerIndex { let mphf1_opt: Option = if need_counts { let p = layer0_dir.join("mphf1.bin"); - p.exists().then(|| Mphf::load_full(&p).ok()).flatten() + p.exists().then(|| unsafe { Mphf::load_full(&p) }.ok()).flatten() } else { None };