Refactor indexing logic and update core dependencies
This commit introduces significant updates to the indexing system, including the introduction of zero-copy types, modifications to the `Mphf` type definition, and new hybrid search strategies. Core algorithms are updated to handle new hashing types and improved error handling during pointer hash construction. Dependency versions are also updated across the board, including new security features and system-level crates.
This commit is contained in:
1 parent
3bd7fd4f41
commit
d686d39506
6 files changed
+185
-205
No files matched your search
Generated
+151
-169
@@ -30,6 +30,12 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aliasable"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd"
|
||||
|
||||
[[package]]
|
||||
name = "allocator-api2"
|
||||
version = "0.2.21"
|
||||
@@ -167,12 +173,6 @@ version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.11.1"
|
||||
@@ -191,6 +191,15 @@ dependencies = [
|
||||
"wyz",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.12.1"
|
||||
@@ -268,11 +277,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cacheline-ef"
|
||||
version = "1.1.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af737c6c59cb018ecbe6472cbdf86d39c59d78252febfe311953a991b6e4ed85"
|
||||
checksum = "6e701af8ea9f58c4ba18793a5d5dffd791cda99bf766976c59c583939d180fd5"
|
||||
dependencies = [
|
||||
"common_traits 0.11.4",
|
||||
"common_traits",
|
||||
"epserde",
|
||||
"mem_dbg",
|
||||
]
|
||||
@@ -312,6 +321,12 @@ version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "cfg_aliases"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
|
||||
|
||||
[[package]]
|
||||
name = "chacha20"
|
||||
version = "0.10.1"
|
||||
@@ -319,7 +334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"cpufeatures 0.3.0",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
@@ -433,17 +448,6 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "common_traits"
|
||||
version = "0.11.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fda9ae1f26adcae83adb2e92f69cf59421f2a277a942f49f8e59f2fcbd7cf062"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"half",
|
||||
"impl-tools 0.10.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "common_traits"
|
||||
version = "0.13.0"
|
||||
@@ -452,7 +456,7 @@ checksum = "d8e0d7d0c5977b1e87f759d406740c38227125f374f54d81bc197e877f2c5f26"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"half",
|
||||
"impl-tools 0.12.0",
|
||||
"impl-tools",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -482,6 +486,15 @@ dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.3.0"
|
||||
@@ -592,6 +605,16 @@ version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.2.2"
|
||||
@@ -645,15 +668,25 @@ dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"crypto-common 0.1.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"block-buffer 0.12.1",
|
||||
"const-oid",
|
||||
"crypto-common",
|
||||
"crypto-common 0.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -662,7 +695,7 @@ version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"bitflags",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
@@ -698,27 +731,27 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "epserde"
|
||||
version = "0.8.0"
|
||||
version = "0.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c40d342ff20a2ce62d9a85ce406e672dfa137f902ac9670034533184f1533976"
|
||||
checksum = "d6cd9f3212ce43e7b594a974c6e8cef4e11c6473d76b7b97cb01c239130789eb"
|
||||
dependencies = [
|
||||
"aliasable",
|
||||
"anyhow",
|
||||
"bitflags 2.11.1",
|
||||
"common_traits 0.11.4",
|
||||
"bitflags",
|
||||
"epserde-derive",
|
||||
"maligned",
|
||||
"maybe-dangling",
|
||||
"mem_dbg",
|
||||
"mmap-rs",
|
||||
"sealed",
|
||||
"sha2",
|
||||
"thiserror 2.0.18",
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "epserde-derive"
|
||||
version = "0.8.0"
|
||||
version = "0.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac80cc78b69765703f48ad93f33b8919cf5d907cda7459ad6ba2919cbbe605dd"
|
||||
checksum = "d73e3083270b2ff144c990e26580374fdfd512f61dd5a12da9b5bd749a7585e0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -806,6 +839,16 @@ dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
@@ -852,6 +895,17 @@ dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"equivalent",
|
||||
"foldhash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
@@ -899,7 +953,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c2e65a48d3b300843ac84a2fe8e166bb5a5b00f30054593bcee8157e4b465fd"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 2.11.1",
|
||||
"bitflags",
|
||||
"derive_more",
|
||||
"errno",
|
||||
"hwlocality-sys",
|
||||
@@ -935,18 +989,6 @@ dependencies = [
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "impl-tools"
|
||||
version = "0.10.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ae95c9095c2f1126d7db785955c73cdc5fc33e7c3fa911bd4a42931672029a7"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"impl-tools-lib 0.11.4",
|
||||
"proc-macro-error2",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "impl-tools"
|
||||
version = "0.12.0"
|
||||
@@ -954,23 +996,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06c8567c75b167cdda5ecee024da9cc9581d41bd52460e2e0e768967972aeef1"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"impl-tools-lib 0.12.0",
|
||||
"impl-tools-lib",
|
||||
"proc-macro-error3",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "impl-tools-lib"
|
||||
version = "0.11.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab699036df31c1f7d3561bfa6e9cb9bc3bb0fd2e2cd9bf121c31cb961d049ddf"
|
||||
dependencies = [
|
||||
"proc-macro-error2",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "impl-tools-lib"
|
||||
version = "0.12.0"
|
||||
@@ -990,7 +1020,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown",
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1043,9 +1073,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.14.0"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
|
||||
checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
@@ -1083,7 +1113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"cpufeatures 0.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1157,7 +1187,7 @@ version = "0.18.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a"
|
||||
dependencies = [
|
||||
"hashbrown",
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1169,12 +1199,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "maligned"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e88c3cbe8288f77f293e48a28b3232e3defd203a6d839fa7f68ea4329e83464"
|
||||
|
||||
[[package]]
|
||||
name = "matchers"
|
||||
version = "0.2.0"
|
||||
@@ -1195,22 +1219,30 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mem_dbg"
|
||||
version = "0.3.4"
|
||||
name = "maybe-dangling"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "728cc9dc97593cd22f7bc81fbef70a2d391d7a9a855e7d658b653318124a6cf0"
|
||||
checksum = "59dbb09ed53f8e4f314e353dc6c1853ae5b4c480a668a422657804a544ea9f65"
|
||||
|
||||
[[package]]
|
||||
name = "mem_dbg"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b48a1086c746f4ee6ca5cb0acf856a14709bc4d2d20e03db150a12ddf2269e6d"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"maligned",
|
||||
"aliasable",
|
||||
"bitflags",
|
||||
"hashbrown 0.16.1",
|
||||
"maybe-dangling",
|
||||
"mem_dbg-derive",
|
||||
"mmap-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mem_dbg-derive"
|
||||
version = "0.2.1"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d84f40c93b0508d5565db79a814d02d5b2545967205ce44be211592aafa34d6c"
|
||||
checksum = "eb910efe8da52f13da727170e352e50a1764579a6fb1065d00d9556da19c79ac"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1232,15 +1264,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
@@ -1253,17 +1276,17 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mmap-rs"
|
||||
version = "0.6.1"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86968d85441db75203c34deefd0c88032f275aaa85cee19a1dcfff6ae9df56da"
|
||||
checksum = "4ecce9d566cb9234ae3db9e249c8b55665feaaf32b0859ff1e27e310d2beb3d8"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"bitflags",
|
||||
"combine",
|
||||
"libc",
|
||||
"mach2",
|
||||
"nix",
|
||||
"sysctl",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror 2.0.18",
|
||||
"widestring",
|
||||
"windows 0.48.0",
|
||||
]
|
||||
@@ -1326,15 +1349,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.26.4"
|
||||
version = "0.30.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
|
||||
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"memoffset",
|
||||
"pin-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1386,7 +1408,7 @@ dependencies = [
|
||||
name = "obicompactvec"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"common_traits 0.13.0",
|
||||
"common_traits",
|
||||
"memmap2",
|
||||
"ndarray",
|
||||
"rayon",
|
||||
@@ -1399,7 +1421,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"crossbeam-channel",
|
||||
"hashbrown",
|
||||
"hashbrown 0.17.1",
|
||||
"obifastwrite",
|
||||
"obikseq",
|
||||
"rayon",
|
||||
@@ -1773,7 +1795,7 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"bitflags",
|
||||
"dispatch2",
|
||||
"objc2",
|
||||
]
|
||||
@@ -1790,7 +1812,7 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"bitflags",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
@@ -1884,12 +1906,6 @@ version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||
|
||||
[[package]]
|
||||
name = "pin-utils"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.33"
|
||||
@@ -1949,14 +1965,10 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-error-attr2"
|
||||
version = "2.0.0"
|
||||
name = "prefetch-index"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
checksum = "9057806a8d77d67bccdc0f542db43737a6f19ada3efab2adc63277feea27310f"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-error-attr3"
|
||||
@@ -1968,17 +1980,6 @@ dependencies = [
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-error2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
|
||||
dependencies = [
|
||||
"proc-macro-error-attr2",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-error3"
|
||||
version = "3.1.0"
|
||||
@@ -2001,30 +2002,25 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ptr_hash"
|
||||
version = "1.1.0"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b4e4fb9c4c2ba3e5b060f53ef46afd3de37345b08e3ec0f2c65e0ca1d57ccbd"
|
||||
checksum = "b22cf1be774892680e056d3387db0dcaa4fd46964b270671ef661e127186b969"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitvec",
|
||||
"cacheline-ef",
|
||||
"clap",
|
||||
"colored",
|
||||
"common_traits 0.11.4",
|
||||
"epserde",
|
||||
"epserde-derive",
|
||||
"fastrand",
|
||||
"fxhash",
|
||||
"itertools 0.14.0",
|
||||
"lazy_static",
|
||||
"itertools 0.15.0",
|
||||
"log",
|
||||
"mem_dbg",
|
||||
"rand 0.9.4",
|
||||
"rand_chacha 0.9.0",
|
||||
"prefetch-index",
|
||||
"rand 0.10.2",
|
||||
"rand_chacha 0.10.0",
|
||||
"rayon",
|
||||
"rdst",
|
||||
"rustc-hash",
|
||||
"serde",
|
||||
"sucds",
|
||||
"tempfile",
|
||||
"xxhash-rust",
|
||||
@@ -2097,16 +2093,6 @@ dependencies = [
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
|
||||
dependencies = [
|
||||
"rand_chacha 0.9.0",
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.10.2"
|
||||
@@ -2130,12 +2116,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.9.0"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
||||
checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.9.5",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2147,15 +2133,6 @@ dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
|
||||
dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.10.1"
|
||||
@@ -2215,7 +2192,7 @@ version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2278,7 +2255,7 @@ checksum = "d9776093b7ca170454ab1406954f7b7d97a57c51dc6c0642957fb2ef25c2d399"
|
||||
dependencies = [
|
||||
"bytecheck",
|
||||
"bytes",
|
||||
"hashbrown",
|
||||
"hashbrown 0.17.1",
|
||||
"indexmap",
|
||||
"munge",
|
||||
"ptr_meta",
|
||||
@@ -2300,12 +2277,6 @@ dependencies = [
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.4.1"
|
||||
@@ -2321,7 +2292,7 @@ version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
@@ -2463,13 +2434,24 @@ dependencies = [
|
||||
"unsafe-libyaml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.2.17",
|
||||
"digest 0.10.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha3"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1"
|
||||
dependencies = [
|
||||
"digest",
|
||||
"digest 0.11.3",
|
||||
"keccak",
|
||||
]
|
||||
|
||||
@@ -2592,11 +2574,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sysctl"
|
||||
version = "0.5.5"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"
|
||||
checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"bitflags",
|
||||
"byteorder",
|
||||
"enum-as-inner",
|
||||
"libc",
|
||||
|
||||
@@ -12,9 +12,9 @@ obicompactvec = { path = "../obicompactvec" }
|
||||
obidebruinj = { path = "../obidebruinj" }
|
||||
obipipeline = { path = "../obipipeline" }
|
||||
obikentropy = { path = "../obikentropy" }
|
||||
cacheline-ef = "1.1"
|
||||
epserde = "0.8"
|
||||
ptr_hash = "1.1"
|
||||
cacheline-ef = "2.0"
|
||||
epserde = "0.13"
|
||||
ptr_hash = { version = "2.1", features = ["epserde"] }
|
||||
niffler = "3.0.0"
|
||||
memmap2 = "0.9.10"
|
||||
ndarray = "0.17"
|
||||
|
||||
@@ -8,7 +8,7 @@ use epserde::prelude::*;
|
||||
use obicompactvec::LayerMeta;
|
||||
use obikseq::CanonicalKmer;
|
||||
use obiskio::{CanonicalKmerIter, UnitigFileReader, UnitigFileWriter, build_unitig_idx};
|
||||
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64};
|
||||
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int};
|
||||
|
||||
use crate::index::error::{OKIError, OKIResult};
|
||||
use crate::layer::evidence::{Evidence, EvidenceWriter};
|
||||
@@ -20,12 +20,10 @@ pub(crate) const UNITIGS_FILE: &str = "unitigs.bin";
|
||||
pub(crate) const EVIDENCE_FILE: &str = "evidence.bin";
|
||||
pub(crate) const FINGERPRINT_FILE: &str = "fingerprint.bin";
|
||||
|
||||
/// Owned MPHF — used only at build time (construction + store).
|
||||
pub(crate) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
|
||||
|
||||
/// Zero-copy MPHF for querying — ε-deserialized view into a memory-mapped file.
|
||||
/// `MemCase` owns the mmap backing; `'static` is sound because MemCase pins the memory.
|
||||
type MphfEps = PtrHash<u64, CubicEps, CachelineEfVec<&'static [CachelineEf]>, Xx64, &'static [u8]>;
|
||||
/// The layer MPHF: `Xxh3Int` hasher, `CubicEps` buckets, multi-part (`SINGLE_PART = false`),
|
||||
/// minimal (`REMAP = true`). Built owned (construction + store); queried through
|
||||
/// `MemCase<Mphf>::uncase()`, the ε-copy view into the memory-mapped file.
|
||||
pub(crate) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||
|
||||
// ── LayerEvidence ─────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -158,7 +156,7 @@ pub struct MphfLayer {
|
||||
/// not a bug).
|
||||
unitigs: Arc<UnitigFileReader>,
|
||||
|
||||
mphf: MemCase<MphfEps>,
|
||||
mphf: MemCase<Mphf>,
|
||||
ev: OpenLayerEvidence,
|
||||
n: usize,
|
||||
}
|
||||
@@ -171,7 +169,7 @@ impl MphfLayer {
|
||||
/// not stored: `KmerLayer`, one level up, is what owns a layer's
|
||||
/// directory as a persistent field, not this type.
|
||||
pub fn open(dir: &Path) -> OKIResult<Self> {
|
||||
let mphf: MemCase<MphfEps> = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty())
|
||||
let mphf: MemCase<Mphf> = unsafe { Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) }
|
||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||
let unitigs = Arc::new(UnitigFileReader::open(&dir.join(UNITIGS_FILE))?);
|
||||
let ev = LayerEvidence::at(dir)?.open()?;
|
||||
@@ -191,7 +189,7 @@ impl MphfLayer {
|
||||
/// - Hybrid: fingerprint check (fast path), zero false positives via `find_strict`.
|
||||
#[inline]
|
||||
pub fn find(&self, kmer: CanonicalKmer) -> Option<usize> {
|
||||
let slot = self.mphf.index(&kmer.raw());
|
||||
let slot = self.mphf.uncase().index(&kmer.raw());
|
||||
if slot >= self.n {
|
||||
return None;
|
||||
}
|
||||
@@ -221,7 +219,7 @@ impl MphfLayer {
|
||||
/// - Approx: O(n) sequential scan of `unitigs.bin` to confirm the kmer
|
||||
/// that owns the slot, then exact comparison.
|
||||
pub fn find_strict(&self, kmer: CanonicalKmer) -> Option<usize> {
|
||||
let slot = self.mphf.index(&kmer.raw());
|
||||
let slot = self.mphf.uncase().index(&kmer.raw());
|
||||
if slot >= self.n {
|
||||
return None;
|
||||
}
|
||||
@@ -243,7 +241,7 @@ impl MphfLayer {
|
||||
// the second file-open + mmap + kmer-count scan a fresh
|
||||
// `open_sequential` call would repeat on every call.
|
||||
for (stored, _, _) in self.unitigs.iter_indexed_canonical_kmers() {
|
||||
if self.mphf.index(&stored.raw()) == slot {
|
||||
if self.mphf.uncase().index(&stored.raw()) == slot {
|
||||
return if stored == kmer { Some(slot) } else { None };
|
||||
}
|
||||
}
|
||||
@@ -274,7 +272,7 @@ impl MphfLayer {
|
||||
/// when the kmer is actually present in the layer; callers that need an
|
||||
/// existence test should use [`find`](Self::find) instead.
|
||||
pub fn hash(&self, kmer: CanonicalKmer) -> usize {
|
||||
self.mphf.index(&kmer.raw())
|
||||
self.mphf.uncase().index(&kmer.raw())
|
||||
}
|
||||
|
||||
/// Batch raw MPHF lookup: kmers → slots.
|
||||
@@ -283,7 +281,7 @@ impl MphfLayer {
|
||||
/// input slice. Like [`index`](Self::index), no membership check is
|
||||
/// performed.
|
||||
pub fn hash_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
|
||||
kmers.iter().map(|k| self.mphf.index(&k.raw())).collect()
|
||||
kmers.iter().map(|k| self.mphf.uncase().index(&k.raw())).collect()
|
||||
}
|
||||
|
||||
/// Iterate over all canonical kmers in the layer, in deterministic order.
|
||||
@@ -426,11 +424,11 @@ impl Iterator for KmerBatchIter {
|
||||
///
|
||||
/// Use this when the caller guarantees that all queried kmers are in the MPHF
|
||||
/// domain (e.g. when iterating the source's own unitigs during merge).
|
||||
pub struct MphfOnly(MemCase<MphfEps>);
|
||||
pub struct MphfOnly(MemCase<Mphf>);
|
||||
|
||||
impl MphfOnly {
|
||||
pub fn open(dir: &Path) -> OKIResult<Self> {
|
||||
let mphf: MemCase<MphfEps> = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty())
|
||||
let mphf: MemCase<Mphf> = unsafe { Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) }
|
||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||
Ok(Self(mphf))
|
||||
}
|
||||
@@ -438,7 +436,7 @@ impl MphfOnly {
|
||||
/// Return the slot for `kmer`. Only valid when `kmer` is in the MPHF domain.
|
||||
#[inline]
|
||||
pub fn index(&self, kmer: CanonicalKmer) -> usize {
|
||||
self.0.index(&kmer.raw())
|
||||
self.0.uncase().index(&kmer.raw())
|
||||
}
|
||||
|
||||
/// Batch form of [`index`](Self::index) — one MPHF lookup pass over the
|
||||
@@ -468,7 +466,7 @@ impl MphfLayer {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE))
|
||||
let mphf: Mphf = unsafe { Mphf::load_full(&dir.join(MPHF_FILE)) }
|
||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||
|
||||
let mut ev = EvidenceWriter::new(n);
|
||||
@@ -512,7 +510,7 @@ impl MphfLayer {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE))
|
||||
let mphf: Mphf = unsafe { Mphf::load_full(&dir.join(MPHF_FILE)) }
|
||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||
|
||||
let mut fw = FingerprintVecWriter::new(n, b);
|
||||
@@ -552,9 +550,9 @@ impl MphfLayer {
|
||||
|
||||
// ── Empty layer ───────────────────────────────────────────────────────
|
||||
if n == 0 {
|
||||
let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::<CubicEps>::default())
|
||||
let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::<CubicEps>::default_balanced())
|
||||
.ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?;
|
||||
mphf.store(&dir.join(MPHF_FILE))
|
||||
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
|
||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||
match mode {
|
||||
IndexMode::Exact | IndexMode::Hybrid { .. } => {
|
||||
@@ -576,9 +574,9 @@ impl MphfLayer {
|
||||
let mphf: Mphf = Mphf::new_from_par_iter(
|
||||
n,
|
||||
keys.map(|k| k.raw()).par_bridge(),
|
||||
PtrHashParams::<CubicEps>::default(),
|
||||
PtrHashParams::<CubicEps>::default_balanced(),
|
||||
);
|
||||
mphf.store(&dir.join(MPHF_FILE))
|
||||
unsafe { mphf.store(&dir.join(MPHF_FILE)) }
|
||||
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
|
||||
|
||||
// ── Pass 2: fill evidence files + callback ────────────────────────────
|
||||
|
||||
@@ -14,9 +14,9 @@ obicompactvec = { path = "../obicompactvec" }
|
||||
obipipeline = { path = "../obipipeline" }
|
||||
obiread = { path = "../obiread" }
|
||||
obiskbuilder = { path = "../obiskbuilder" }
|
||||
cacheline-ef = "1.1"
|
||||
epserde = "0.8"
|
||||
ptr_hash = "1.1"
|
||||
cacheline-ef = "2.0"
|
||||
epserde = "0.13"
|
||||
ptr_hash = { version = "2.1", features = ["epserde"] }
|
||||
niffler = "3.0.0"
|
||||
memmap2 = "0.9.10"
|
||||
rayon = "1"
|
||||
|
||||
@@ -8,12 +8,12 @@ use epserde::ser::Serialize as EpSerialize;
|
||||
use memmap2::Mmap;
|
||||
use obicompactvec::PersistentCompactIntVecBuilder;
|
||||
use obiskio::{SKFileReader, SKResult};
|
||||
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64};
|
||||
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xxh3Int};
|
||||
use tracing::debug;
|
||||
|
||||
use super::kmer_sort::sort_unique_kmers;
|
||||
|
||||
pub(super) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
|
||||
pub(super) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||
|
||||
fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
|
||||
let file = fs::File::open(unique_path)?;
|
||||
@@ -24,7 +24,7 @@ fn build_mphf(unique_path: &Path, f0: usize) -> io::Result<Mphf> {
|
||||
// Sequential constructor: the outer par_iter over partitions already saturates
|
||||
// the Rayon pool. new_from_par_iter would get no additional threads and adds
|
||||
// coordination overhead. try_new accesses the same mmap'd pages at zero extra cost.
|
||||
Mphf::try_new(kmers, PtrHashParams::<CubicEps>::default())
|
||||
Mphf::try_new(kmers, PtrHashParams::<CubicEps>::default_balanced())
|
||||
.ok_or_else(|| io::Error::other("ptr_hash construction failed"))
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ pub(super) fn count_partition(dir: &Path, dedup_path: &Path, chunk_kmers: usize)
|
||||
)
|
||||
.map_err(io::Error::other)?;
|
||||
|
||||
EpSerialize::store(&mphf, &dir.join("mphf1.bin"))
|
||||
unsafe { EpSerialize::store(&mphf, &dir.join("mphf1.bin")) }
|
||||
.map_err(|e| io::Error::other(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -33,9 +33,9 @@ use obikindex::layer::IndexMode;
|
||||
use obikindex::layer::{KmerLayer, TypedLayer};
|
||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
||||
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
||||
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xxh3Int};
|
||||
|
||||
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
|
||||
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xxh3Int, Vec<u8>, false, true>;
|
||||
|
||||
pub(crate) trait PrivateBuilder {
|
||||
/// Mark scatter as complete (`IndexState::Scattered`).
|
||||
@@ -157,7 +157,7 @@ impl PrivateBuilder for KmerIndex {
|
||||
|
||||
let mphf1_opt: Option<Mphf> = if need_counts {
|
||||
let p = layer0_dir.join("mphf1.bin");
|
||||
p.exists().then(|| Mphf::load_full(&p).ok()).flatten()
|
||||
p.exists().then(|| unsafe { Mphf::load_full(&p) }.ok()).flatten()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user