Refactor index metadata loading to use buffered I/O

This change replaces unbuffered file reads with buffered I/O operations to improve performance during index metadata loading. This includes implementing new methods for reading index metadata files and wrapping file handles in `BufReader` to optimize subsequent reads.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:17:51 +02:00
1 parent 093d81c359
commit d32d86193f
4 files changed
+71 -6

No files matched your search

+4 -2
View File
@@ -280,8 +280,10 @@ impl IndexMeta {
// ── private ──────────────────────────────────────────────────────────────
fn read_full(root: &Path) -> io::Result<IndexMetadata> {
let file = fs::File::open(root.join(META_FILENAME))?;
serde_json::from_reader(file).map_err(io::Error::other)
// One `read`, then parse from memory: `from_reader` on a bare `File`
// issues one syscall per byte (≈ 0.35 s for a 91-genome `index.meta`).
let bytes = fs::read(root.join(META_FILENAME))?;
serde_json::from_slice(&bytes).map_err(io::Error::other)
}
fn write_full(&self, on_disk: &IndexMetadata) -> io::Result<()> {
+4 -2
View File
@@ -28,8 +28,10 @@ impl SKFileMeta {
/// Read the sidecar for `sk_path`. Returns `None` if the sidecar is absent.
pub fn read(sk_path: &Path) -> SKResult<Option<Self>> {
let path = Self::sidecar_path(sk_path);
match std::fs::File::open(&path) {
Ok(f) => Ok(Some(serde_json::from_reader(f).map_err(|e| SKError::InvalidData {
// Read whole, parse from memory: `from_reader` on a bare `File` is one
// syscall per byte.
match std::fs::read(&path) {
Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes).map_err(|e| SKError::InvalidData {
context: "SKFileMeta sidecar",
detail: e.to_string(),
})?)),
+6 -2
View File
@@ -19,8 +19,12 @@ impl SKFileReader {
/// Open a file for reading. Format is auto-detected from magic bytes.
pub fn open<P: AsRef<Path>>(path: P) -> SKResult<Self> {
let path = path.as_ref().to_owned();
let (reader, _fmt) =
niffler::send::get_reader(Box::new(File::open(&path)?))?;
// Buffered: for an uncompressed file niffler hands back the bare
// `File` (after the sniffed bytes), and `read_superkmer` reads each
// field with its own small `read_exact` — one syscall each otherwise.
let (reader, _fmt) = niffler::send::get_reader(Box::new(
std::io::BufReader::with_capacity(1 << 16, File::open(&path)?),
))?;
Ok(Self { path, reader })
}