Refactor query system for performance and memory reduction

This change introduces dynamic chunk sizing based on input size estimates, ensuring memory constraints are respected while allowing for smaller chunks that yield significant speed improvements. Additionally, index structures were optimized, resulting in drastic memory footprint reduction and faster query execution times.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:21:38 +02:00
1 parent 025c40c0db
commit f9edefd135
6 files changed
+152 -3

No files matched your search

+24 -1
View File
@@ -192,17 +192,40 @@ pub fn run(args: QueryArgs) {
let overhead_multiplier =
BASE_OVERHEAD + n_genomes as u64 * BYTES_PER_KMER_PER_GENOME * detail_factor;
// Parallelism: a chunk is one worker's unit of work, so an input smaller
// than `n_workers` chunks leaves cores idle (two 48 MB files at the
// memory-derived 256 MiB cap = 2 chunks = 2 busy cores; measured 11.6 s
// against 2.5 s with 8 MiB chunks, see `DevDocMD/architecture/query.md`).
// When the input size can be estimated, aim for `CHUNKS_PER_WORKER` chunks
// per worker and never go below `MIN_CHUNK`. Stdin and URLs have no
// known size: the memory-derived size stands.
const CHUNKS_PER_WORKER: u64 = 2;
const MIN_CHUNK: u64 = 4 * 1024 * 1024;
let input_bytes: Option<u64> = args
.inputs
.iter()
.map(|p| obiread::estimated_decompressed_len(p))
.sum::<Option<u64>>();
let chunk_bytes = args
.chunk_size
.map(|mb| mb * 1024 * 1024)
.unwrap_or_else(|| {
let avail = available_memory_bytes();
let computed = avail / (n_workers as u64 * overhead_multiplier * SAFETY_FACTOR);
computed.clamp(4 * 1024 * 1024, 256 * 1024 * 1024) as usize
let by_memory = computed.clamp(4 * 1024 * 1024, 256 * 1024 * 1024);
match input_bytes {
Some(total) => {
let by_parallelism = (total / (n_workers as u64 * CHUNKS_PER_WORKER)).max(MIN_CHUNK);
by_memory.min(by_parallelism) as usize
}
None => by_memory as usize,
}
});
debug!(
chunk_bytes,
input_bytes,
n_genomes,
detail = args.detail,
overhead_multiplier,
+1 -1
View File
@@ -25,7 +25,7 @@ pub use normalize::{normalize_fasta_chunk, normalize_fastq_chunk, normalize_sequ
pub use nucstream::{NucPage, NucPageCursor, open_nuc_stream};
pub use path_iterator::{PathIter, path_iter};
pub use peakreader::PeekReader;
pub use xopen::xopen;
pub use xopen::{estimated_decompressed_len, xopen};
/// Default read block size: 1 MiB.
pub const DEFAULT_BLOCK_SIZE: usize = 1024 * 1024;
+38
View File
@@ -0,0 +1,38 @@
use super::*;
use std::io::Write;
fn temp_file(name: &str, content: &[u8]) -> std::path::PathBuf {
let path = std::env::temp_dir().join(format!("obiread_xopen_{}_{name}", std::process::id()));
std::fs::File::create(&path).unwrap().write_all(content).unwrap();
path
}
#[test]
fn estimate_plain_file_is_its_length() {
let p = temp_file("plain", b">s\nACGT\n");
assert_eq!(estimated_decompressed_len(p.to_str().unwrap()), Some(8));
std::fs::remove_file(p).unwrap();
}
#[test]
fn estimate_scales_compressed_files_by_magic_not_extension() {
// Only the magic bytes matter to the estimate (the content is never decoded).
let mut gz = vec![0x1f, 0x8b, 0x08, 0, 0, 0];
gz.resize(100, 0);
let p = temp_file("looks_plain.txt", &gz);
assert_eq!(estimated_decompressed_len(p.to_str().unwrap()), Some(300));
std::fs::remove_file(p).unwrap();
let mut zst = vec![0x28, 0xb5, 0x2f, 0xfd, 0, 0];
zst.resize(50, 0);
let p = temp_file("zst", &zst);
assert_eq!(estimated_decompressed_len(p.to_str().unwrap()), Some(200));
std::fs::remove_file(p).unwrap();
}
#[test]
fn estimate_is_unknown_for_streams_and_missing_files() {
assert_eq!(estimated_decompressed_len("-"), None);
assert_eq!(estimated_decompressed_len("https://example.org/x.fa.gz"), None);
assert_eq!(estimated_decompressed_len("/nonexistent/obiread/file"), None);
}
+35
View File
@@ -46,6 +46,37 @@ pub fn xopen(source: &str) -> io::Result<MimeTypeGuesser<Box<dyn Read + Send>>>
Ok(MimeTypeGuesser::new(open_raw(source)?))
}
/// Rough size, in bytes, of what [`xopen`] would yield for a local file:
/// its length, multiplied by a typical expansion factor when the magic bytes
/// say it is compressed (gzip ≈ 3×, as measured on FASTQ; bzip2, xz and zstd
/// taken as 4×). Meant for sizing work (chunk sizes), not for allocation —
/// the real ratio varies by a factor of two or more with the content.
///
/// `None` when the size cannot be known without reading the stream: stdin
/// (`-`), an HTTP(S) URL, or a file that cannot be opened.
pub fn estimated_decompressed_len(source: &str) -> Option<u64> {
if source == "-" || source.starts_with("http://") || source.starts_with("https://") {
return None;
}
let expanded = expand_tilde(source);
let mut file = File::open(expanded.as_ref()).ok()?;
let len = file.metadata().ok()?.len();
let mut magic = [0u8; 6];
let n = file.read(&mut magic).ok()?;
let magic = &magic[..n];
let factor = if magic.starts_with(&[0x1f, 0x8b]) {
3
} else if magic.starts_with(b"BZh")
|| magic.starts_with(&[0xfd, b'7', b'z', b'X', b'Z', 0x00])
|| magic.starts_with(&[0x28, 0xb5, 0x2f, 0xfd])
{
4
} else {
1
};
Some(len * factor)
}
// ── internal helpers ──────────────────────────────────────────────────────────
fn http_reader(url: &str) -> io::Result<Box<dyn Read + Send>> {
@@ -71,3 +102,7 @@ fn expand_tilde(path: &str) -> std::borrow::Cow<'_, str> {
}
std::borrow::Cow::Borrowed(path)
}
#[cfg(test)]
#[path = "tests/xopen.rs"]
mod tests;