Refactor index metadata loading to use buffered I/O
This change replaces unbuffered file reads with buffered I/O operations to improve performance during index metadata loading. This includes implementing new methods for reading index metadata files and wrapping file handles in `BufReader` to optimize subsequent reads.
This commit is contained in:
1 parent
093d81c359
commit
d32d86193f
4 files changed
+71
-6
No files matched your search
@@ -280,8 +280,10 @@ impl IndexMeta {
|
||||
// ── private ──────────────────────────────────────────────────────────────
|
||||
|
||||
fn read_full(root: &Path) -> io::Result<IndexMetadata> {
|
||||
let file = fs::File::open(root.join(META_FILENAME))?;
|
||||
serde_json::from_reader(file).map_err(io::Error::other)
|
||||
// One `read`, then parse from memory: `from_reader` on a bare `File`
|
||||
// issues one syscall per byte (≈ 0.35 s for a 91-genome `index.meta`).
|
||||
let bytes = fs::read(root.join(META_FILENAME))?;
|
||||
serde_json::from_slice(&bytes).map_err(io::Error::other)
|
||||
}
|
||||
|
||||
fn write_full(&self, on_disk: &IndexMetadata) -> io::Result<()> {
|
||||
|
||||
@@ -28,8 +28,10 @@ impl SKFileMeta {
|
||||
/// Read the sidecar for `sk_path`. Returns `None` if the sidecar is absent.
|
||||
pub fn read(sk_path: &Path) -> SKResult<Option<Self>> {
|
||||
let path = Self::sidecar_path(sk_path);
|
||||
match std::fs::File::open(&path) {
|
||||
Ok(f) => Ok(Some(serde_json::from_reader(f).map_err(|e| SKError::InvalidData {
|
||||
// Read whole, parse from memory: `from_reader` on a bare `File` is one
|
||||
// syscall per byte.
|
||||
match std::fs::read(&path) {
|
||||
Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes).map_err(|e| SKError::InvalidData {
|
||||
context: "SKFileMeta sidecar",
|
||||
detail: e.to_string(),
|
||||
})?)),
|
||||
|
||||
@@ -19,8 +19,12 @@ impl SKFileReader {
|
||||
/// Open a file for reading. Format is auto-detected from magic bytes.
|
||||
pub fn open<P: AsRef<Path>>(path: P) -> SKResult<Self> {
|
||||
let path = path.as_ref().to_owned();
|
||||
let (reader, _fmt) =
|
||||
niffler::send::get_reader(Box::new(File::open(&path)?))?;
|
||||
// Buffered: for an uncompressed file niffler hands back the bare
|
||||
// `File` (after the sniffed bytes), and `read_superkmer` reads each
|
||||
// field with its own small `read_exact` — one syscall each otherwise.
|
||||
let (reader, _fmt) = niffler::send::get_reader(Box::new(
|
||||
std::io::BufReader::with_capacity(1 << 16, File::open(&path)?),
|
||||
))?;
|
||||
Ok(Self { path, reader })
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user