From d32d86193fac9ccfe33df12da650d4a26adc57ae Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Thu, 15 Oct 2026 10:00:23 +0200 Subject: [PATCH] Refactor index metadata loading to use buffered I/O This change replaces unbuffered file reads with buffered I/O operations to improve performance during index metadata loading. This includes implementing new methods for reading index metadata files and wrapping file handles in `BufReader` to optimize subsequent reads. --- DevDocMD/implementation/mphf_batch_queries.md | 57 +++++++++++++++++++ src/obikindex/src/index/meta.rs | 6 +- src/obiskio/src/meta.rs | 6 +- src/obiskio/src/reader.rs | 8 ++- 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/DevDocMD/implementation/mphf_batch_queries.md b/DevDocMD/implementation/mphf_batch_queries.md index 4901f10c..55a3e473 100644 --- a/DevDocMD/implementation/mphf_batch_queries.md +++ b/DevDocMD/implementation/mphf_batch_queries.md @@ -210,3 +210,60 @@ The `query` stage is ≈ 50 % of wall time; the rest (≈ 13 s) is opening the i | after | **0.26–0.81 s, 0.04 GB** | **10.7 s, 3.6 GB**, `query` stage 9.95 s | Tests: `cargo test --workspace` green; `scripts/smoke_test_index.sh` PASS; `bench_mphf.sh` (index, merge, query ×4, filter, convert round trip, phylo) completes with all output digests identical to the baseline run, i.e. no failure from files mapped while rewritten (the risk of mapping instead of copying: a reader holding a `.idx` map while the same path is truncated and rewritten would fault). The per-step ratios of that run vs the earlier baseline mix this change with the machine moving from battery to AC power and must not be read as its effect. + +## 15. Unbuffered reads: `index.meta` and superkmer files (2026-10-15) + +Follow-up of §14 (search for the same family of problems: file content handled inefficiently at open). Static search over `src/` for `fs::read`, `read_to_end`, `load_full`, `from_reader`, bare `File` readers and mmap `to_vec`: every persistent structure of `obicompactvec`, `obikindex` and the `obikphylo` annexes is already mmapped; the two real findings were *unbuffered* readers. + +**1. `IndexMeta::read_full` (`obikindex/src/index/meta.rs`)** passed a bare `File` to `serde_json::from_reader`, which issues **one `read` syscall per byte**. `index.meta` of the 91-genome index is 20 KB and is read afresh by every `state()`/`genomes()` call (stateless design, §obikindex audit), so each call cost ≈ 0.35 s here (macOS). Profile of `obikmer dump --head 10` (`sample`): ≈ 20 000 samples in `read`, called from `Bytes::next` ← `serde_json` ← `IndexMeta`. Fix: `fs::read` then `serde_json::from_slice`. **`dump --head 10`: 9.0–10.0 s (91–102 s of system CPU) → 0.16–0.84 s**; the same change applies to `SKFileMeta::read` (`obiskio/src/meta.rs`, per-superkmer-file sidecars). + +**2. `SKFileReader::open` (`obiskio/src/reader.rs`)**: for an uncompressed file `niffler::get_reader` returns the sniffed bytes chained to the bare `File`, and `read_superkmer` reads each field with its own small `read_exact`. Fix: wrap the file in a 64 KiB `BufReader` before `niffler`. Effect on `index` (E. coli, 4 interleaved runs each): `count_kmer` 0.51–0.56 s → 0.42–0.50 s (≈ −12 %), total wall unchanged (≈ 1.95 s). Small, low risk, kept. + +Tests: `cargo test --workspace` 549 passed / 0 failed; `scripts/smoke_test_index.sh` PASS; query output on the 91-genome index identical (md5 `e9e7c35b`). + +**Probes of other commands on the 91-genome index (180 GB, 256 partitions)**: `phylo --distance jaccard` peaks at 155–190 MB of footprint while running and finishes in < 30 s; `utils --stats` 130–150 MB. `filter` (and by the same check `select`, `merge`) refused the index — "source index is not fully built" — with the unmodified baseline binary too: this index's `index.meta` predates the `state` field, so they could not be probed. + +**Observations not investigated**: `index` on E. coli shows ≈ 9 s of *system* CPU in 2 s of wall time (both before and after these changes), consistent with the `swtch_pri` (yield) spinning seen in the `index` profile of §11 rather than with I/O; and wall times of the same query on the same binary varied 4.8 s ↔ 10.7 s ↔ 43 s depending on the state of the page cache, so only same-session comparisons are meaningful. + +## 16. Profile of `index` on E. coli reads (2026-10-15) + +`sample` (1 ms stacks) of `obikmer index` on the E. coli K-12 simulated reads (2 × 15x, 16 threads, one run; the `index` stage lasted 2.4 s under the sampler vs ≈ 0.9 s normally, so proportions are approximate). Stage walls: scatter 0.40 s, dereplicate 0.14, count_kmer 0.60, index 2.36. + +**Threads are mostly idle.** Of ≈ 30 800 thread-samples: 24 900 blocked or yielding, 5 900 active. The ≈ 9 s of *system* CPU per run (§15) is `swtch_pri` (`sched_yield`) — 4 550 samples: ≈ 70 % rayon workers spinning for work (`WorkerThread` wait loop), ≈ 28 % `crossbeam` bounded-channel send/recv spinning, the rest in `obidebruinj` thread-locals. It is waiting, not I/O: no `read` samples remain. It costs no wall time on an otherwise idle machine, but burns CPU (visible in `user + sys` accounting and on a shared 192-core machine). + +**Where the active CPU goes** (share of the 5 900 active samples): + +| | share | +|---|---| +| xxh3 hashing of `CanonicalKmer` for the de Bruijn graph's `HashMap` (callers: `count_neighbors` 791, `hash_one` 358, `walk` 285, `unitig_iter` 252, `push` 164, `unitig_nucleotides` 75, rehash 68; plus `xxh3_stateful_update` 300) | ≈ 34 % | +| `debruijn::count_neighbors` itself (inclusive of its hashing ≈ 2 300 samples: `compute_degrees_and_mark_starts`) | 12.5 % own | +| `ptr_hash::build` (MPHF construction) | 4.8 % | +| crossbeam channel send | 4.8 % | +| `open` / `close` / `write` (many small files over 256 partitions) | ≈ 9 % | +| debruijn `unitig_iter` / `walk` own time | 8.3 % | + +So at this scale the cost of `index` is the **de Bruijn graph's hash table** (several lookups of neighbouring k-mers per k-mer, each hashed with xxh3 and probably missing the cache), not the MPHF passes this document started from (`SlottedKmers` ≈ 2 %, §11). Not measured: what fraction of those lookups miss the cache, and the same profile on a prod-size partition. + +## 17. Profile of `index` on a skim: *Vaccinium vitis-idaea*, 2.1 GB, `--partitions 16` (2026-10-15) + +One run, `sample` windows of 5 s every ~17 s over the whole run (the sampler slows the process; walls below include it): scatter 19.7 s, dereplicate 8.0 s, count_kmer 25.2 s, **index 139.1 s**; total 192 s real, 1830 s user, **225 s system** (12 % of user, against ≈ 90 % on E. coli, §16), peak footprint 17.8 GB. + +**Index stage** (samples taken fully inside it, 91 000 active vs 187 000 idle/yielding thread-samples): + +| share of active CPU | | +|---|---| +| xxh3 hashing of kmers (de Bruijn `HashMap`) | 32.7 % | +| `debruijn::walk` | 12.4 % | +| `debruijn::count_neighbors` | 10.7 % | +| `GraphDeBruijn::push` | 6.9 % | +| `crossbeam` channel | 6.6 % | +| `hashbrown` table operations | 5.0 % | +| `debruijn::unitig_iter` | 4.8 % | +| memcpy/memset, malloc/free, file I/O, SipHash | ≈ 7 % | +| other | 13.7 % | + +- **The de Bruijn graph is ≈ 72 % of the active CPU of `index`; the MPHF construction does not appear (< 1 %)**, and `SlottedKmers`/evidence filling is negligible. The conclusion of §16 holds at 100× the E. coli scale. +- **The stage is poorly parallel**: on average ≈ 5 of 16 threads are active (33 % of thread-samples); `swtch_pri` (spin/yield) is 7 % of all thread-samples. With 16 partitions of unequal size the tail partitions run alone. The 12 % system CPU is the same spinning as §16, at a smaller relative weight. +- **count_kmer** (one window): sorting 40 %, file I/O 18 %, malloc/free 9 %, memcpy 6 %. **scatter** (one window, 32 % active): `RollingStat::push` 32 %, `EntropyTracker::push` 23 % + `normalized_entropy` 5 %, super-kmer stream 10 %, gzip decode ≈ 9 %. + +Not established: the cache-miss rate of the graph's table (stalls are attributed to whichever instruction waits, so "hashing 33 %" may include time waiting for the hasher's operands or surrounding loads — a counter-based profile is needed to separate compute from memory stalls), and whether a cheaper hash function for `u64` kmers or fewer lookups per kmer would change the picture. diff --git a/src/obikindex/src/index/meta.rs b/src/obikindex/src/index/meta.rs index 83da9cad..9cb67621 100644 --- a/src/obikindex/src/index/meta.rs +++ b/src/obikindex/src/index/meta.rs @@ -280,8 +280,10 @@ impl IndexMeta { // ── private ────────────────────────────────────────────────────────────── fn read_full(root: &Path) -> io::Result { - let file = fs::File::open(root.join(META_FILENAME))?; - serde_json::from_reader(file).map_err(io::Error::other) + // One `read`, then parse from memory: `from_reader` on a bare `File` + // issues one syscall per byte (≈ 0.35 s for a 91-genome `index.meta`). + let bytes = fs::read(root.join(META_FILENAME))?; + serde_json::from_slice(&bytes).map_err(io::Error::other) } fn write_full(&self, on_disk: &IndexMetadata) -> io::Result<()> { diff --git a/src/obiskio/src/meta.rs b/src/obiskio/src/meta.rs index 87f2aeec..4b8983e8 100644 --- a/src/obiskio/src/meta.rs +++ b/src/obiskio/src/meta.rs @@ -28,8 +28,10 @@ impl SKFileMeta { /// Read the sidecar for `sk_path`. Returns `None` if the sidecar is absent. pub fn read(sk_path: &Path) -> SKResult> { let path = Self::sidecar_path(sk_path); - match std::fs::File::open(&path) { - Ok(f) => Ok(Some(serde_json::from_reader(f).map_err(|e| SKError::InvalidData { + // Read whole, parse from memory: `from_reader` on a bare `File` is one + // syscall per byte. + match std::fs::read(&path) { + Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes).map_err(|e| SKError::InvalidData { context: "SKFileMeta sidecar", detail: e.to_string(), })?)), diff --git a/src/obiskio/src/reader.rs b/src/obiskio/src/reader.rs index 9694f182..e39e5169 100644 --- a/src/obiskio/src/reader.rs +++ b/src/obiskio/src/reader.rs @@ -19,8 +19,12 @@ impl SKFileReader { /// Open a file for reading. Format is auto-detected from magic bytes. pub fn open>(path: P) -> SKResult { let path = path.as_ref().to_owned(); - let (reader, _fmt) = - niffler::send::get_reader(Box::new(File::open(&path)?))?; + // Buffered: for an uncompressed file niffler hands back the bare + // `File` (after the sniffed bytes), and `read_superkmer` reads each + // field with its own small `read_exact` — one syscall each otherwise. + let (reader, _fmt) = niffler::send::get_reader(Box::new( + std::io::BufReader::with_capacity(1 << 16, File::open(&path)?), + ))?; Ok(Self { path, reader }) }