From af525cd268b3579833926e0d5ae106e0296d926b Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Wed, 18 Nov 2026 09:18:57 +0100 Subject: [PATCH] feat: switch musl builds to jemalloc allocator for stability Update Cargo.lock to remove unused dependencies and update tikv-jemallocator. Configure src/obikmer/Cargo.toml to use jemalloc for static musl builds, addressing fragmentation issues with the default multi-threaded allocator. Update main.rs to conditionally set Jemalloc as the global allocator when targeting musl. - Remove unused dependencies from Cargo.lock: anes, block-pseudorand, chiapos-chacha8, criterion, criterion-plot, itertools (and its either dep), nanorand, hermit-abi, is-terminal, plotters ecosystem, tinytemplate deps, voracious_radix_sort rayon dep, and web-sys deps. - Update tikv-jemallocator version from 0.5.4 to 0.7.0 in Cargo.lock. - Add tikv-jemallocator = "0.7.0" to musl-specific dependencies in src/obikmer/Cargo.toml. - Set Jemalloc as the global allocator in main.rs for static musl builds. --- src/Cargo.lock | 169 ++-------------------------------------- src/obikmer/Cargo.toml | 7 ++ src/obikmer/src/main.rs | 5 ++ 3 files changed, 20 insertions(+), 161 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index 4c09a281..13668f52 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -29,12 +29,6 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - [[package]] name = "anstream" version = "1.0.0" @@ -196,16 +190,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "block-pseudorand" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2097358495d244a0643746f4d13eedba4608137008cf9dec54e53a3b700115a6" -dependencies = [ - "chiapos-chacha8", - "nanorand", -] - [[package]] name = "bpaf" version = "0.9.27" @@ -325,15 +309,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "chiapos-chacha8" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33f8be573a85f6c2bc1b8e43834c07e32f95e489b914bf856c0549c3c269cd0a" -dependencies = [ - "rayon", -] - [[package]] name = "ciborium" version = "0.2.2" @@ -500,42 +475,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "is-terminal", - "itertools 0.10.5", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools 0.10.5", -] - [[package]] name = "criterion2" version = "3.0.4" @@ -911,12 +850,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - [[package]] name = "http" version = "1.4.2" @@ -1032,32 +965,12 @@ dependencies = [ "cfb", ] -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.15.0" @@ -1298,12 +1211,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "nanorand" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958" - [[package]] name = "ndarray" version = "0.17.2" @@ -1558,6 +1465,7 @@ dependencies = [ "rayon", "serde", "serde_yaml", + "tikv-jemallocator", "tracing", "tracing-subscriber", ] @@ -1884,34 +1792,6 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - [[package]] name = "portable-atomic" version = "1.13.1" @@ -1985,7 +1865,7 @@ dependencies = [ "epserde-derive", "fastrand", "fxhash", - "itertools 0.15.0", + "itertools", "log", "mem_dbg", "prefetch-index", @@ -2145,17 +2025,13 @@ checksum = "e6724c78e033e1c4155b4d3f76593d09ad384739c6986c1addc2bd2f55b1aefe" [[package]] name = "rdst" -version = "0.20.14" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7970b4e577b76a96d5e56b5f6662b66d1a4e1f5bb026ee118fc31b373c2752" +checksum = "6cf18dd479f856ca75267591c062e22b50e2db791157495bc2d533558de7061d" dependencies = [ "arbitrary-chunks", - "block-pseudorand", - "criterion", "partition", "rayon", - "tikv-jemallocator", - "voracious_radix_sort", ] [[package]] @@ -2654,9 +2530,9 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" -version = "0.5.4+5.3.0-patched" +version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" +checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0" dependencies = [ "cc", "libc", @@ -2664,24 +2540,14 @@ dependencies = [ [[package]] name = "tikv-jemallocator" -version = "0.5.4" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" +checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd" dependencies = [ "libc", "tikv-jemalloc-sys", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.13.2" @@ -2869,15 +2735,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "voracious_radix_sort" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446e7ffcb6c27a71d05af7e51ef2ee5b71c48424b122a832f2439651e1914899" -dependencies = [ - "rayon", -] - [[package]] name = "walkdir" version = "2.5.0" @@ -2948,16 +2805,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "web-sys" -version = "0.3.95" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "web-time" version = "1.1.0" diff --git a/src/obikmer/Cargo.toml b/src/obikmer/Cargo.toml index ac394aa1..a430fdd6 100644 --- a/src/obikmer/Cargo.toml +++ b/src/obikmer/Cargo.toml @@ -36,6 +36,13 @@ serde_yaml = "0.9" tracing = "0.1.44" tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } +# musl's built-in allocator fragments badly under the merge's multi-threaded +# allocation pattern (map count and RSS drift, see DevDocMD/implementation/ +# merge_mmap_audit.md). Static musl builds use jemalloc instead; glibc and +# macOS builds keep the system allocator. +[target.'cfg(target_env = "musl")'.dependencies] +tikv-jemallocator = "0.7.0" + [features] default = ["numa"] numa = ["obisys/numa"] diff --git a/src/obikmer/src/main.rs b/src/obikmer/src/main.rs index 422af2d5..f0ff8baa 100644 --- a/src/obikmer/src/main.rs +++ b/src/obikmer/src/main.rs @@ -1,3 +1,8 @@ +// Static musl builds: replace musl's allocator (see obikmer/Cargo.toml). +#[cfg(target_env = "musl")] +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + mod cli; mod cmd;