perf: enable zero-allocation queries and memory-mapped indexes
Introduce zero-allocation row extraction and query result buffers across `obicompactvec` and `obikpartitionner` to eliminate per-kmer heap allocations. Replace in-memory MPHF deserialization with memory-mapped, zero-copy views to reduce runtime memory footprint. Add configurable I/O chunking, a RAM-aware `--chunk-size` parameter, and system memory monitoring via the new `sysinfo` dependency. Re-export `PreloadedIndex` for external consumers.
This commit is contained in:
@@ -4,4 +4,5 @@ version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
libc = "0.2"
|
||||
libc = "0.2"
|
||||
sysinfo = "0.33"
|
||||
|
||||
@@ -2,6 +2,21 @@ use std::fmt;
|
||||
use std::time::Instant;
|
||||
|
||||
use libc::{RUSAGE_SELF, getrusage, rusage, timeval};
|
||||
use sysinfo::System;
|
||||
|
||||
// ── Memory query ──────────────────────────────────────────────────────────────
|
||||
|
||||
/// Returns the number of bytes available for allocation on this machine.
|
||||
///
|
||||
/// On macOS, `available_memory()` can return 0 when the memory compressor
|
||||
/// inflates the page count; in that case we fall back to half of total memory.
|
||||
pub fn available_memory_bytes() -> u64 {
|
||||
let sys = System::new_all();
|
||||
match sys.available_memory() {
|
||||
0 => sys.total_memory() / 2,
|
||||
n => n,
|
||||
}
|
||||
}
|
||||
|
||||
// ── raw helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
Reference in New Issue
Block a user